The Tor Browser / file revision

 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/

 /* This Source Code Form is subject to the terms of the Mozilla Public

  * License, v. 2.0. If a copy of the MPL was not distributed with this file,

  * You can obtain one at http://mozilla.org/MPL/2.0/. */

 #ifndef MOZILLA_MEDIASTREAMGRAPHIMPL_H_

 #define MOZILLA_MEDIASTREAMGRAPHIMPL_H_

 #include "MediaStreamGraph.h"

 #include "mozilla/Monitor.h"

 #include "mozilla/TimeStamp.h"

 #include "nsIMemoryReporter.h"

 #include "nsIThread.h"

 #include "nsIRunnable.h"

 #include "Latency.h"

 #include "mozilla/WeakPtr.h"

 namespace mozilla {

 template <typename T>

 class LinkedList;

 class AudioMixer;

 /**

  * Assume we can run an iteration of the MediaStreamGraph loop in this much time

  * or less.

  * We try to run the control loop at this rate.

  */

 static const int MEDIA_GRAPH_TARGET_PERIOD_MS = 10;

 /**

  * Assume that we might miss our scheduled wakeup of the MediaStreamGraph by

  * this much.

  */

 static const int SCHEDULE_SAFETY_MARGIN_MS = 10;

 /**

  * Try have this much audio buffered in streams and queued to the hardware.

  * The maximum delay to the end of the next control loop

  * is 2*MEDIA_GRAPH_TARGET_PERIOD_MS + SCHEDULE_SAFETY_MARGIN_MS.

  * There is no point in buffering more audio than this in a stream at any

  * given time (until we add processing).

  * This is not optimal yet.

  */

 static const int AUDIO_TARGET_MS = 2*MEDIA_GRAPH_TARGET_PERIOD_MS +

     SCHEDULE_SAFETY_MARGIN_MS;

 /**

  * Try have this much video buffered. Video frames are set

  * near the end of the iteration of the control loop. The maximum delay

  * to the setting of the next video frame is 2*MEDIA_GRAPH_TARGET_PERIOD_MS +

  * SCHEDULE_SAFETY_MARGIN_MS. This is not optimal yet.

  */

 static const int VIDEO_TARGET_MS = 2*MEDIA_GRAPH_TARGET_PERIOD_MS +

     SCHEDULE_SAFETY_MARGIN_MS;

 /**

  * A per-stream update message passed from the media graph thread to the

  * main thread.

  */

 struct StreamUpdate {

   int64_t mGraphUpdateIndex;

   nsRefPtr<MediaStream> mStream;

   StreamTime mNextMainThreadCurrentTime;

   bool mNextMainThreadFinished;

 };

 /**

  * This represents a message passed from the main thread to the graph thread.

  * A ControlMessage always has a weak reference a particular affected stream.

  */

 class ControlMessage {

 public:

   explicit ControlMessage(MediaStream* aStream) : mStream(aStream)

   {

     MOZ_COUNT_CTOR(ControlMessage);

   }

   // All these run on the graph thread

   virtual ~ControlMessage()

   {

     MOZ_COUNT_DTOR(ControlMessage);

   }

   // Do the action of this message on the MediaStreamGraph thread. Any actions

   // affecting graph processing should take effect at mStateComputedTime.

   // All stream data for times < mStateComputedTime has already been

   // computed.

   virtual void Run() = 0;

   // When we're shutting down the application, most messages are ignored but

   // some cleanup messages should still be processed (on the main thread).

   // This must not add new control messages to the graph.

   virtual void RunDuringShutdown() {}

   MediaStream* GetStream() { return mStream; }

 protected:

   // We do not hold a reference to mStream. The graph will be holding

   // a reference to the stream until the Destroy message is processed. The

   // last message referencing a stream is the Destroy message for that stream.

   MediaStream* mStream;

 };

 /**

  * The implementation of a media stream graph. This class is private to this

  * file. It's not in the anonymous namespace because MediaStream needs to

  * be able to friend it.

  *

  * Currently we have one global instance per process, and one per each

  * OfflineAudioContext object.

  */

 class MediaStreamGraphImpl : public MediaStreamGraph,

                              public nsIMemoryReporter {

 public:

   NS_DECL_ISUPPORTS

   NS_DECL_NSIMEMORYREPORTER

   /**

    * Set aRealtime to true in order to create a MediaStreamGraph which provides

    * support for real-time audio and video.  Set it to false in order to create

    * a non-realtime instance which just churns through its inputs and produces

    * output.  Those objects currently only support audio, and are used to

    * implement OfflineAudioContext.  They do not support MediaStream inputs.

    */

   explicit MediaStreamGraphImpl(bool aRealtime, TrackRate aSampleRate);

   /**

    * Unregisters memory reporting and deletes this instance. This should be

    * called instead of calling the destructor directly.

    */

   void Destroy();

   // Main thread only.

   /**

    * This runs every time we need to sync state from the media graph thread

    * to the main thread while the main thread is not in the middle

    * of a script. It runs during a "stable state" (per HTML5) or during

    * an event posted to the main thread.

    */

   void RunInStableState();

   /**

    * Ensure a runnable to run RunInStableState is posted to the appshell to

    * run at the next stable state (per HTML5).

    * See EnsureStableStateEventPosted.

    */

   void EnsureRunInStableState();

   /**

    * Called to apply a StreamUpdate to its stream.

    */

   void ApplyStreamUpdate(StreamUpdate* aUpdate);

   /**

    * Append a ControlMessage to the message queue. This queue is drained

    * during RunInStableState; the messages will run on the graph thread.

    */

   void AppendMessage(ControlMessage* aMessage);

   /**

    * Make this MediaStreamGraph enter forced-shutdown state. This state

    * will be noticed by the media graph thread, which will shut down all streams

    * and other state controlled by the media graph thread.

    * This is called during application shutdown.

    */

   void ForceShutDown();

   /**

    * Shutdown() this MediaStreamGraph's threads and return when they've shut down.

    */

   void ShutdownThreads();

   /**

    * Called before the thread runs.

    */

   void Init();

   // The following methods run on the graph thread (or possibly the main thread if

   // mLifecycleState > LIFECYCLE_RUNNING)

   /**

    * Runs main control loop on the graph thread. Normally a single invocation

    * of this runs for the entire lifetime of the graph thread.

    */

   void RunThread();

   /**

    * Call this to indicate that another iteration of the control loop is

    * required on its regular schedule. The monitor must not be held.

    */

   void EnsureNextIteration();

   /**

    * As above, but with the monitor already held.

    */

   void EnsureNextIterationLocked(MonitorAutoLock& aLock);

   /**

    * Call this to indicate that another iteration of the control loop is

    * required immediately. The monitor must already be held.

    */

   void EnsureImmediateWakeUpLocked(MonitorAutoLock& aLock);

   /**

    * Ensure there is an event posted to the main thread to run RunInStableState.

    * mMonitor must be held.

    * See EnsureRunInStableState

    */

   void EnsureStableStateEventPosted();

   /**

    * Generate messages to the main thread to update it for all state changes.

    * mMonitor must be held.

    */

   void PrepareUpdatesToMainThreadState(bool aFinalUpdate);

   /**

    * Returns false if there is any stream that has finished but not yet finished

    * playing out.

    */

   bool AllFinishedStreamsNotified();

   /**

    * If we are rendering in non-realtime mode, we don't want to send messages to

    * the main thread at each iteration for performance reasons. We instead

    * notify the main thread at the same rate

    */

   bool ShouldUpdateMainThread();

   // The following methods are the various stages of RunThread processing.

   /**

    * Compute a new current time for the graph and advance all on-graph-thread

    * state to the new current time.

    */

   void UpdateCurrentTime();

   /**

    * Update the consumption state of aStream to reflect whether its data

    * is needed or not.

    */

   void UpdateConsumptionState(SourceMediaStream* aStream);

   /**

    * Extract any state updates pending in aStream, and apply them.

    */

   void ExtractPendingInput(SourceMediaStream* aStream,

                            GraphTime aDesiredUpToTime,

                            bool* aEnsureNextIteration);

   /**

    * Update "have enough data" flags in aStream.

    */

   void UpdateBufferSufficiencyState(SourceMediaStream* aStream);

   /*

    * If aStream hasn't already been ordered, push it onto aStack and order

    * its children.

    */

   void UpdateStreamOrderForStream(mozilla::LinkedList<MediaStream>* aStack,

                                   already_AddRefed<MediaStream> aStream);

   /**

    * Mark aStream and all its inputs (recursively) as consumed.

    */

   static void MarkConsumed(MediaStream* aStream);

   /**

    * Sort mStreams so that every stream not in a cycle is after any streams

    * it depends on, and every stream in a cycle is marked as being in a cycle.

    * Also sets mIsConsumed on every stream.

    */

   void UpdateStreamOrder();

   /**

    * Compute the blocking states of streams from mStateComputedTime

    * until the desired future time aEndBlockingDecisions.

    * Updates mStateComputedTime and sets MediaStream::mBlocked

    * for all streams.

    */

   void RecomputeBlocking(GraphTime aEndBlockingDecisions);

   // The following methods are used to help RecomputeBlocking.

   /**

    * If aStream isn't already in aStreams, add it and recursively call

    * AddBlockingRelatedStreamsToSet on all the streams whose blocking

    * status could depend on or affect the state of aStream.

    */

   void AddBlockingRelatedStreamsToSet(nsTArray<MediaStream*>* aStreams,

                                       MediaStream* aStream);

   /**

    * Mark a stream blocked at time aTime. If this results in decisions that need

    * to be revisited at some point in the future, *aEnd will be reduced to the

    * first time in the future to recompute those decisions.

    */

   void MarkStreamBlocking(MediaStream* aStream);

   /**

    * Recompute blocking for the streams in aStreams for the interval starting at aTime.

    * If this results in decisions that need to be revisited at some point

    * in the future, *aEnd will be reduced to the first time in the future to

    * recompute those decisions.

    */

   void RecomputeBlockingAt(const nsTArray<MediaStream*>& aStreams,

                            GraphTime aTime, GraphTime aEndBlockingDecisions,

                            GraphTime* aEnd);

   /**

    * Produce data for all streams >= aStreamIndex for the given time interval.

    * Advances block by block, each iteration producing data for all streams

    * for a single block.

    * This is called whenever we have an AudioNodeStream in the graph.

    */

   void ProduceDataForStreamsBlockByBlock(uint32_t aStreamIndex,

                                          TrackRate aSampleRate,

                                          GraphTime aFrom,

                                          GraphTime aTo);

   /**

    * Returns true if aStream will underrun at aTime for its own playback.

    * aEndBlockingDecisions is when we plan to stop making blocking decisions.

    * *aEnd will be reduced to the first time in the future to recompute these

    * decisions.

    */

   bool WillUnderrun(MediaStream* aStream, GraphTime aTime,

                     GraphTime aEndBlockingDecisions, GraphTime* aEnd);

   /**

    * Given a graph time aTime, convert it to a stream time taking into

    * account the time during which aStream is scheduled to be blocked.

    */

   StreamTime GraphTimeToStreamTime(MediaStream* aStream, GraphTime aTime);

   /**

    * Given a graph time aTime, convert it to a stream time taking into

    * account the time during which aStream is scheduled to be blocked, and

    * when we don't know whether it's blocked or not, we assume it's not blocked.

    */

   StreamTime GraphTimeToStreamTimeOptimistic(MediaStream* aStream, GraphTime aTime);

   enum {

     INCLUDE_TRAILING_BLOCKED_INTERVAL = 0x01

   };

   /**

    * Given a stream time aTime, convert it to a graph time taking into

    * account the time during which aStream is scheduled to be blocked.

    * aTime must be <= mStateComputedTime since blocking decisions

    * are only known up to that point.

    * If aTime is exactly at the start of a blocked interval, then the blocked

    * interval is included in the time returned if and only if

    * aFlags includes INCLUDE_TRAILING_BLOCKED_INTERVAL.

    */

   GraphTime StreamTimeToGraphTime(MediaStream* aStream, StreamTime aTime,

                                   uint32_t aFlags = 0);

   /**

    * Get the current audio position of the stream's audio output.

    */

   GraphTime GetAudioPosition(MediaStream* aStream);

   /**

    * Call NotifyHaveCurrentData on aStream's listeners.

    */

   void NotifyHasCurrentData(MediaStream* aStream);

   /**

    * If aStream needs an audio stream but doesn't have one, create it.

    * If aStream doesn't need an audio stream but has one, destroy it.

    */

   void CreateOrDestroyAudioStreams(GraphTime aAudioOutputStartTime,

                                    MediaStream* aStream);

   /**

    * Queue audio (mix of stream audio and silence for blocked intervals)

    * to the audio output stream. Returns the number of frames played.

    */

   TrackTicks PlayAudio(MediaStream* aStream, GraphTime aFrom, GraphTime aTo);

   /**

    * Set the correct current video frame for stream aStream.

    */

   void PlayVideo(MediaStream* aStream);

   /**

    * No more data will be forthcoming for aStream. The stream will end

    * at the current buffer end point. The StreamBuffer's tracks must be

    * explicitly set to finished by the caller.

    */

   void FinishStream(MediaStream* aStream);

   /**

    * Compute how much stream data we would like to buffer for aStream.

    */

   StreamTime GetDesiredBufferEnd(MediaStream* aStream);

   /**

    * Returns true when there are no active streams.

    */

   bool IsEmpty() { return mStreams.IsEmpty() && mPortCount == 0; }

   // For use by control messages, on graph thread only.

   /**

    * Identify which graph update index we are currently processing.

    */

   int64_t GetProcessingGraphUpdateIndex() { return mProcessingGraphUpdateIndex; }

   /**

    * Add aStream to the graph and initializes its graph-specific state.

    */

   void AddStream(MediaStream* aStream);

   /**

    * Remove aStream from the graph. Ensures that pending messages about the

    * stream back to the main thread are flushed.

    */

   void RemoveStream(MediaStream* aStream);

   /**

    * Remove aPort from the graph and release it.

    */

   void DestroyPort(MediaInputPort* aPort);

   /**

    * Mark the media stream order as dirty.

    */

   void SetStreamOrderDirty()

   {

     mStreamOrderDirty = true;

   }

   /**

    * Pause all AudioStreams being written to by MediaStreams

    */

   void PauseAllAudioOutputs();

   /**

    * Resume all AudioStreams being written to by MediaStreams

    */

   void ResumeAllAudioOutputs();

   TrackRate AudioSampleRate() { return mSampleRate; }

   // Data members

   /**

    * Media graph thread.

    * Readonly after initialization on the main thread.

    */

   nsCOMPtr<nsIThread> mThread;

   // The following state is managed on the graph thread only, unless

   // mLifecycleState > LIFECYCLE_RUNNING in which case the graph thread

   // is not running and this state can be used from the main thread.

   nsTArray<nsRefPtr<MediaStream> > mStreams;

   /**

    * mOldStreams is used as temporary storage for streams when computing the

    * order in which we compute them.

    */

   nsTArray<nsRefPtr<MediaStream> > mOldStreams;

   /**

    * The current graph time for the current iteration of the RunThread control

    * loop.

    */

   GraphTime mCurrentTime;

   /**

    * Blocking decisions and all stream contents have been computed up to this

    * time. The next batch of updates from the main thread will be processed

    * at this time. Always >= mCurrentTime.

    */

   GraphTime mStateComputedTime;

   /**

    * This is only used for logging.

    */

   TimeStamp mInitialTimeStamp;

   /**

    * The real timestamp of the latest run of UpdateCurrentTime.

    */

   TimeStamp mCurrentTimeStamp;

   /**

    * Date of the last time we updated the main thread with the graph state.

    */

   TimeStamp mLastMainThreadUpdate;

   /**

    * Which update batch we are currently processing.

    */

   int64_t mProcessingGraphUpdateIndex;

   /**

    * Number of active MediaInputPorts

    */

   int32_t mPortCount;

   // mMonitor guards the data below.

   // MediaStreamGraph normally does its work without holding mMonitor, so it is

   // not safe to just grab mMonitor from some thread and start monkeying with

   // the graph. Instead, communicate with the graph thread using provided

   // mechanisms such as the ControlMessage queue.

   Monitor mMonitor;

   // Data guarded by mMonitor (must always be accessed with mMonitor held,

   // regardless of the value of mLifecycleState.

   /**

    * State to copy to main thread

    */

   nsTArray<StreamUpdate> mStreamUpdates;

   /**

    * Runnables to run after the next update to main thread state.

    */

   nsTArray<nsCOMPtr<nsIRunnable> > mUpdateRunnables;

   struct MessageBlock {

     int64_t mGraphUpdateIndex;

     nsTArray<nsAutoPtr<ControlMessage> > mMessages;

   };

   /**

    * A list of batches of messages to process. Each batch is processed

    * as an atomic unit.

    */

   nsTArray<MessageBlock> mMessageQueue;

   /**

    * This enum specifies where this graph is in its lifecycle. This is used

    * to control shutdown.

    * Shutdown is tricky because it can happen in two different ways:

    * 1) Shutdown due to inactivity. RunThread() detects that it has no

    * pending messages and no streams, and exits. The next RunInStableState()

    * checks if there are new pending messages from the main thread (true only

    * if new stream creation raced with shutdown); if there are, it revives

    * RunThread(), otherwise it commits to shutting down the graph. New stream

    * creation after this point will create a new graph. An async event is

    * dispatched to Shutdown() the graph's threads and then delete the graph

    * object.

    * 2) Forced shutdown at application shutdown, or completion of a

    * non-realtime graph. A flag is set, RunThread() detects the flag and

    * exits, the next RunInStableState() detects the flag, and dispatches the

    * async event to Shutdown() the graph's threads. However the graph object

    * is not deleted. New messages for the graph are processed synchronously on

    * the main thread if necessary. When the last stream is destroyed, the

    * graph object is deleted.

    */

   enum LifecycleState {

     // The graph thread hasn't started yet.

     LIFECYCLE_THREAD_NOT_STARTED,

     // RunThread() is running normally.

     LIFECYCLE_RUNNING,

     // In the following states, the graph thread is not running so

     // all "graph thread only" state in this class can be used safely

     // on the main thread.

     // RunThread() has exited and we're waiting for the next

     // RunInStableState(), at which point we can clean up the main-thread

     // side of the graph.

     LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP,

     // RunInStableState() posted a ShutdownRunnable, and we're waiting for it

     // to shut down the graph thread(s).

     LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN,

     // Graph threads have shut down but we're waiting for remaining streams

     // to be destroyed. Only happens during application shutdown and on

     // completed non-realtime graphs, since normally we'd only shut down a

     // realtime graph when it has no streams.

     LIFECYCLE_WAITING_FOR_STREAM_DESTRUCTION

   };

   LifecycleState mLifecycleState;

   /**

    * This enum specifies the wait state of the graph thread.

    */

   enum WaitState {

     // RunThread() is running normally

     WAITSTATE_RUNNING,

     // RunThread() is paused waiting for its next iteration, which will

     // happen soon

     WAITSTATE_WAITING_FOR_NEXT_ITERATION,

     // RunThread() is paused indefinitely waiting for something to change

     WAITSTATE_WAITING_INDEFINITELY,

     // Something has signaled RunThread() to wake up immediately,

     // but it hasn't done so yet

     WAITSTATE_WAKING_UP

   };

   WaitState mWaitState;

   /**

    * The graph should stop processing at or after this time.

    */

   GraphTime mEndTime;

   /**

    * Sample rate at which this graph runs. For real time graphs, this is

    * the rate of the audio mixer. For offline graphs, this is the rate specified

    * at construction.

    */

   TrackRate mSampleRate;

   /**

    * True when another iteration of the control loop is required.

    */

   bool mNeedAnotherIteration;

   /**

    * True when we need to do a forced shutdown during application shutdown.

    */

   bool mForceShutDown;

   /**

    * True when we have posted an event to the main thread to run

    * RunInStableState() and the event hasn't run yet.

    */

   bool mPostedRunInStableStateEvent;

   // Main thread only

   /**

    * Messages posted by the current event loop task. These are forwarded to

    * the media graph thread during RunInStableState. We can't forward them

    * immediately because we want all messages between stable states to be

    * processed as an atomic batch.

    */

   nsTArray<nsAutoPtr<ControlMessage> > mCurrentTaskMessageQueue;

   /**

    * True when RunInStableState has determined that mLifecycleState is >

    * LIFECYCLE_RUNNING. Since only the main thread can reset mLifecycleState to

    * LIFECYCLE_RUNNING, this can be relied on to not change unexpectedly.

    */

   bool mDetectedNotRunning;

   /**

    * True when a stable state runner has been posted to the appshell to run

    * RunInStableState at the next stable state.

    */

   bool mPostedRunInStableState;

   /**

    * True when processing real-time audio/video.  False when processing non-realtime

    * audio.

    */

   bool mRealtime;

   /**

    * True when a non-realtime MediaStreamGraph has started to process input.  This

    * value is only accessed on the main thread.

    */

   bool mNonRealtimeProcessing;

   /**

    * True when a change has happened which requires us to recompute the stream

    * blocking order.

    */

   bool mStreamOrderDirty;

   /**

    * Hold a ref to the Latency logger

    */

   nsRefPtr<AsyncLatencyLogger> mLatencyLog;

   /**

    * If this is not null, all the audio output for the MSG will be mixed down.

    */

   nsAutoPtr<AudioMixer> mMixer;

 private:

   virtual ~MediaStreamGraphImpl();

   MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf)

   /**

    * Used to signal that a memory report has been requested.

    */

   Monitor mMemoryReportMonitor;

   /**

    * This class uses manual memory management, and all pointers to it are raw

    * pointers. However, in order for it to implement nsIMemoryReporter, it needs

    * to implement nsISupports and so be ref-counted. So it maintains a single

    * nsRefPtr to itself, giving it a ref-count of 1 during its entire lifetime,

    * and Destroy() nulls this self-reference in order to trigger self-deletion.

    */

   nsRefPtr<MediaStreamGraphImpl> mSelfRef;

   /**

    * Used to pass memory report information across threads.

    */

   nsTArray<AudioNodeSizes> mAudioStreamSizes;

   /**

    * Indicates that the MSG thread should gather data for a memory report.

    */

   bool mNeedsMemoryReport;

 #ifdef DEBUG

   /**

    * Used to assert when AppendMessage() runs ControlMessages synchronously.

    */

   bool mCanRunMessagesSynchronously;

 #endif

 };

 }

 #endif /* MEDIASTREAMGRAPHIMPL_H_ */