The Tor Browser: content/media/MediaStreamGraphImpl.h@97036ab72558 (annotated)

content/media/MediaStreamGraphImpl.h@97036ab72558 (annotated)

content/media/MediaStreamGraphImpl.h

Tue, 06 Jan 2015 21:39:09 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Tue, 06 Jan 2015 21:39:09 +0100
branch: TOR_BUG_9701
changeset 8: 97036ab72558
permissions: -rw-r--r--

Conditionally force memory storage according to privacy.thirdparty.isolate;
This solves Tor bug #9701, complying with disk avoidance documented in
https://www.torproject.org/projects/torbrowser/design/#disk-avoidance.

 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 #ifndef MOZILLA_MEDIASTREAMGRAPHIMPL_H_
 #define MOZILLA_MEDIASTREAMGRAPHIMPL_H_
 #include "MediaStreamGraph.h"
 #include "mozilla/Monitor.h"
 #include "mozilla/TimeStamp.h"
 #include "nsIMemoryReporter.h"
 #include "nsIThread.h"
 #include "nsIRunnable.h"
 #include "Latency.h"
 #include "mozilla/WeakPtr.h"
 namespace mozilla {
 template <typename T>
 class LinkedList;
 class AudioMixer;
 /**
  * Assume we can run an iteration of the MediaStreamGraph loop in this much time
  * or less.
  * We try to run the control loop at this rate.
  */
 static const int MEDIA_GRAPH_TARGET_PERIOD_MS = 10;
 /**
  * Assume that we might miss our scheduled wakeup of the MediaStreamGraph by
  * this much.
  */
 static const int SCHEDULE_SAFETY_MARGIN_MS = 10;
 /**
  * Try have this much audio buffered in streams and queued to the hardware.
  * The maximum delay to the end of the next control loop
  * is 2*MEDIA_GRAPH_TARGET_PERIOD_MS + SCHEDULE_SAFETY_MARGIN_MS.
  * There is no point in buffering more audio than this in a stream at any
  * given time (until we add processing).
  * This is not optimal yet.
  */
 static const int AUDIO_TARGET_MS = 2*MEDIA_GRAPH_TARGET_PERIOD_MS +
     SCHEDULE_SAFETY_MARGIN_MS;
 /**
  * Try have this much video buffered. Video frames are set
  * near the end of the iteration of the control loop. The maximum delay
  * to the setting of the next video frame is 2*MEDIA_GRAPH_TARGET_PERIOD_MS +
  * SCHEDULE_SAFETY_MARGIN_MS. This is not optimal yet.
  */
 static const int VIDEO_TARGET_MS = 2*MEDIA_GRAPH_TARGET_PERIOD_MS +
     SCHEDULE_SAFETY_MARGIN_MS;
 /**
  * A per-stream update message passed from the media graph thread to the
  * main thread.
  */
 struct StreamUpdate {
   int64_t mGraphUpdateIndex;
   nsRefPtr<MediaStream> mStream;
   StreamTime mNextMainThreadCurrentTime;
   bool mNextMainThreadFinished;
 };
 /**
  * This represents a message passed from the main thread to the graph thread.
  * A ControlMessage always has a weak reference a particular affected stream.
  */
 class ControlMessage {
 public:
   explicit ControlMessage(MediaStream* aStream) : mStream(aStream)
   {
     MOZ_COUNT_CTOR(ControlMessage);
   }
   // All these run on the graph thread
   virtual ~ControlMessage()
   {
     MOZ_COUNT_DTOR(ControlMessage);
   }
   // Do the action of this message on the MediaStreamGraph thread. Any actions
   // affecting graph processing should take effect at mStateComputedTime.
   // All stream data for times < mStateComputedTime has already been
   // computed.
   virtual void Run() = 0;
   // When we're shutting down the application, most messages are ignored but
   // some cleanup messages should still be processed (on the main thread).
   // This must not add new control messages to the graph.
   virtual void RunDuringShutdown() {}
   MediaStream* GetStream() { return mStream; }
 protected:
   // We do not hold a reference to mStream. The graph will be holding
   // a reference to the stream until the Destroy message is processed. The
   // last message referencing a stream is the Destroy message for that stream.
   MediaStream* mStream;
 };
 /**
  * The implementation of a media stream graph. This class is private to this
  * file. It's not in the anonymous namespace because MediaStream needs to
  * be able to friend it.
  *
  * Currently we have one global instance per process, and one per each
  * OfflineAudioContext object.
  */
 class MediaStreamGraphImpl : public MediaStreamGraph,
                              public nsIMemoryReporter {
 public:
   NS_DECL_ISUPPORTS
   NS_DECL_NSIMEMORYREPORTER
   /**
    * Set aRealtime to true in order to create a MediaStreamGraph which provides
    * support for real-time audio and video.  Set it to false in order to create
    * a non-realtime instance which just churns through its inputs and produces
    * output.  Those objects currently only support audio, and are used to
    * implement OfflineAudioContext.  They do not support MediaStream inputs.
    */
   explicit MediaStreamGraphImpl(bool aRealtime, TrackRate aSampleRate);
   /**
    * Unregisters memory reporting and deletes this instance. This should be
    * called instead of calling the destructor directly.
    */
   void Destroy();
   // Main thread only.
   /**
    * This runs every time we need to sync state from the media graph thread
    * to the main thread while the main thread is not in the middle
    * of a script. It runs during a "stable state" (per HTML5) or during
    * an event posted to the main thread.
    */
   void RunInStableState();
   /**
    * Ensure a runnable to run RunInStableState is posted to the appshell to
    * run at the next stable state (per HTML5).
    * See EnsureStableStateEventPosted.
    */
   void EnsureRunInStableState();
   /**
    * Called to apply a StreamUpdate to its stream.
    */
   void ApplyStreamUpdate(StreamUpdate* aUpdate);
   /**
    * Append a ControlMessage to the message queue. This queue is drained
    * during RunInStableState; the messages will run on the graph thread.
    */
   void AppendMessage(ControlMessage* aMessage);
   /**
    * Make this MediaStreamGraph enter forced-shutdown state. This state
    * will be noticed by the media graph thread, which will shut down all streams
    * and other state controlled by the media graph thread.
    * This is called during application shutdown.
    */
   void ForceShutDown();
   /**
    * Shutdown() this MediaStreamGraph's threads and return when they've shut down.
    */
   void ShutdownThreads();
   /**
    * Called before the thread runs.
    */
   void Init();
   // The following methods run on the graph thread (or possibly the main thread if
   // mLifecycleState > LIFECYCLE_RUNNING)
   /**
    * Runs main control loop on the graph thread. Normally a single invocation
    * of this runs for the entire lifetime of the graph thread.
    */
   void RunThread();
   /**
    * Call this to indicate that another iteration of the control loop is
    * required on its regular schedule. The monitor must not be held.
    */
   void EnsureNextIteration();
   /**
    * As above, but with the monitor already held.
    */
   void EnsureNextIterationLocked(MonitorAutoLock& aLock);
   /**
    * Call this to indicate that another iteration of the control loop is
    * required immediately. The monitor must already be held.
    */
   void EnsureImmediateWakeUpLocked(MonitorAutoLock& aLock);
   /**
    * Ensure there is an event posted to the main thread to run RunInStableState.
    * mMonitor must be held.
    * See EnsureRunInStableState
    */
   void EnsureStableStateEventPosted();
   /**
    * Generate messages to the main thread to update it for all state changes.
    * mMonitor must be held.
    */
   void PrepareUpdatesToMainThreadState(bool aFinalUpdate);
   /**
    * Returns false if there is any stream that has finished but not yet finished
    * playing out.
    */
   bool AllFinishedStreamsNotified();
   /**
    * If we are rendering in non-realtime mode, we don't want to send messages to
    * the main thread at each iteration for performance reasons. We instead
    * notify the main thread at the same rate
    */
   bool ShouldUpdateMainThread();
   // The following methods are the various stages of RunThread processing.
   /**
    * Compute a new current time for the graph and advance all on-graph-thread
    * state to the new current time.
    */
   void UpdateCurrentTime();
   /**
    * Update the consumption state of aStream to reflect whether its data
    * is needed or not.
    */
   void UpdateConsumptionState(SourceMediaStream* aStream);
   /**
    * Extract any state updates pending in aStream, and apply them.
    */
   void ExtractPendingInput(SourceMediaStream* aStream,
                            GraphTime aDesiredUpToTime,
                            bool* aEnsureNextIteration);
   /**
    * Update "have enough data" flags in aStream.
    */
   void UpdateBufferSufficiencyState(SourceMediaStream* aStream);
   /*
    * If aStream hasn't already been ordered, push it onto aStack and order
    * its children.
    */
   void UpdateStreamOrderForStream(mozilla::LinkedList<MediaStream>* aStack,
                                   already_AddRefed<MediaStream> aStream);
   /**
    * Mark aStream and all its inputs (recursively) as consumed.
    */
   static void MarkConsumed(MediaStream* aStream);
   /**
    * Sort mStreams so that every stream not in a cycle is after any streams
    * it depends on, and every stream in a cycle is marked as being in a cycle.
    * Also sets mIsConsumed on every stream.
    */
   void UpdateStreamOrder();
   /**
    * Compute the blocking states of streams from mStateComputedTime
    * until the desired future time aEndBlockingDecisions.
    * Updates mStateComputedTime and sets MediaStream::mBlocked
    * for all streams.
    */
   void RecomputeBlocking(GraphTime aEndBlockingDecisions);
   // The following methods are used to help RecomputeBlocking.
   /**
    * If aStream isn't already in aStreams, add it and recursively call
    * AddBlockingRelatedStreamsToSet on all the streams whose blocking
    * status could depend on or affect the state of aStream.
    */
   void AddBlockingRelatedStreamsToSet(nsTArray<MediaStream*>* aStreams,
                                       MediaStream* aStream);
   /**
    * Mark a stream blocked at time aTime. If this results in decisions that need
    * to be revisited at some point in the future, *aEnd will be reduced to the
    * first time in the future to recompute those decisions.
    */
   void MarkStreamBlocking(MediaStream* aStream);
   /**
    * Recompute blocking for the streams in aStreams for the interval starting at aTime.
    * If this results in decisions that need to be revisited at some point
    * in the future, *aEnd will be reduced to the first time in the future to
    * recompute those decisions.
    */
   void RecomputeBlockingAt(const nsTArray<MediaStream*>& aStreams,
                            GraphTime aTime, GraphTime aEndBlockingDecisions,
                            GraphTime* aEnd);
   /**
    * Produce data for all streams >= aStreamIndex for the given time interval.
    * Advances block by block, each iteration producing data for all streams
    * for a single block.
    * This is called whenever we have an AudioNodeStream in the graph.
    */
   void ProduceDataForStreamsBlockByBlock(uint32_t aStreamIndex,
                                          TrackRate aSampleRate,
                                          GraphTime aFrom,
                                          GraphTime aTo);
   /**
    * Returns true if aStream will underrun at aTime for its own playback.
    * aEndBlockingDecisions is when we plan to stop making blocking decisions.
    * *aEnd will be reduced to the first time in the future to recompute these
    * decisions.
    */
   bool WillUnderrun(MediaStream* aStream, GraphTime aTime,
                     GraphTime aEndBlockingDecisions, GraphTime* aEnd);
   /**
    * Given a graph time aTime, convert it to a stream time taking into
    * account the time during which aStream is scheduled to be blocked.
    */
   StreamTime GraphTimeToStreamTime(MediaStream* aStream, GraphTime aTime);
   /**
    * Given a graph time aTime, convert it to a stream time taking into
    * account the time during which aStream is scheduled to be blocked, and
    * when we don't know whether it's blocked or not, we assume it's not blocked.
    */
   StreamTime GraphTimeToStreamTimeOptimistic(MediaStream* aStream, GraphTime aTime);
   enum {
     INCLUDE_TRAILING_BLOCKED_INTERVAL = 0x01
   };
   /**
    * Given a stream time aTime, convert it to a graph time taking into
    * account the time during which aStream is scheduled to be blocked.
    * aTime must be <= mStateComputedTime since blocking decisions
    * are only known up to that point.
    * If aTime is exactly at the start of a blocked interval, then the blocked
    * interval is included in the time returned if and only if
    * aFlags includes INCLUDE_TRAILING_BLOCKED_INTERVAL.
    */
   GraphTime StreamTimeToGraphTime(MediaStream* aStream, StreamTime aTime,
                                   uint32_t aFlags = 0);
   /**
    * Get the current audio position of the stream's audio output.
    */
   GraphTime GetAudioPosition(MediaStream* aStream);
   /**
    * Call NotifyHaveCurrentData on aStream's listeners.
    */
   void NotifyHasCurrentData(MediaStream* aStream);
   /**
    * If aStream needs an audio stream but doesn't have one, create it.
    * If aStream doesn't need an audio stream but has one, destroy it.
    */
   void CreateOrDestroyAudioStreams(GraphTime aAudioOutputStartTime,
                                    MediaStream* aStream);
   /**
    * Queue audio (mix of stream audio and silence for blocked intervals)
    * to the audio output stream. Returns the number of frames played.
    */
   TrackTicks PlayAudio(MediaStream* aStream, GraphTime aFrom, GraphTime aTo);
   /**
    * Set the correct current video frame for stream aStream.
    */
   void PlayVideo(MediaStream* aStream);
   /**
    * No more data will be forthcoming for aStream. The stream will end
    * at the current buffer end point. The StreamBuffer's tracks must be
    * explicitly set to finished by the caller.
    */
   void FinishStream(MediaStream* aStream);
   /**
    * Compute how much stream data we would like to buffer for aStream.
    */
   StreamTime GetDesiredBufferEnd(MediaStream* aStream);
   /**
    * Returns true when there are no active streams.
    */
   bool IsEmpty() { return mStreams.IsEmpty() && mPortCount == 0; }
   // For use by control messages, on graph thread only.
   /**
    * Identify which graph update index we are currently processing.
    */
   int64_t GetProcessingGraphUpdateIndex() { return mProcessingGraphUpdateIndex; }
   /**
    * Add aStream to the graph and initializes its graph-specific state.
    */
   void AddStream(MediaStream* aStream);
   /**
    * Remove aStream from the graph. Ensures that pending messages about the
    * stream back to the main thread are flushed.
    */
   void RemoveStream(MediaStream* aStream);
   /**
    * Remove aPort from the graph and release it.
    */
   void DestroyPort(MediaInputPort* aPort);
   /**
    * Mark the media stream order as dirty.
    */
   void SetStreamOrderDirty()
   {
     mStreamOrderDirty = true;
   }
   /**
    * Pause all AudioStreams being written to by MediaStreams
    */
   void PauseAllAudioOutputs();
   /**
    * Resume all AudioStreams being written to by MediaStreams
    */
   void ResumeAllAudioOutputs();
   TrackRate AudioSampleRate() { return mSampleRate; }
   // Data members
   /**
    * Media graph thread.
    * Readonly after initialization on the main thread.
    */
   nsCOMPtr<nsIThread> mThread;
   // The following state is managed on the graph thread only, unless
   // mLifecycleState > LIFECYCLE_RUNNING in which case the graph thread
   // is not running and this state can be used from the main thread.
   nsTArray<nsRefPtr<MediaStream> > mStreams;
   /**
    * mOldStreams is used as temporary storage for streams when computing the
    * order in which we compute them.
    */
   nsTArray<nsRefPtr<MediaStream> > mOldStreams;
   /**
    * The current graph time for the current iteration of the RunThread control
    * loop.
    */
   GraphTime mCurrentTime;
   /**
    * Blocking decisions and all stream contents have been computed up to this
    * time. The next batch of updates from the main thread will be processed
    * at this time. Always >= mCurrentTime.
    */
   GraphTime mStateComputedTime;
   /**
    * This is only used for logging.
    */
   TimeStamp mInitialTimeStamp;
   /**
    * The real timestamp of the latest run of UpdateCurrentTime.
    */
   TimeStamp mCurrentTimeStamp;
   /**
    * Date of the last time we updated the main thread with the graph state.
    */
   TimeStamp mLastMainThreadUpdate;
   /**
    * Which update batch we are currently processing.
    */
   int64_t mProcessingGraphUpdateIndex;
   /**
    * Number of active MediaInputPorts
    */
   int32_t mPortCount;
   // mMonitor guards the data below.
   // MediaStreamGraph normally does its work without holding mMonitor, so it is
   // not safe to just grab mMonitor from some thread and start monkeying with
   // the graph. Instead, communicate with the graph thread using provided
   // mechanisms such as the ControlMessage queue.
   Monitor mMonitor;
   // Data guarded by mMonitor (must always be accessed with mMonitor held,
   // regardless of the value of mLifecycleState.
   /**
    * State to copy to main thread
    */
   nsTArray<StreamUpdate> mStreamUpdates;
   /**
    * Runnables to run after the next update to main thread state.
    */
   nsTArray<nsCOMPtr<nsIRunnable> > mUpdateRunnables;
   struct MessageBlock {
     int64_t mGraphUpdateIndex;
     nsTArray<nsAutoPtr<ControlMessage> > mMessages;
   };
   /**
    * A list of batches of messages to process. Each batch is processed
    * as an atomic unit.
    */
   nsTArray<MessageBlock> mMessageQueue;
   /**
    * This enum specifies where this graph is in its lifecycle. This is used
    * to control shutdown.
    * Shutdown is tricky because it can happen in two different ways:
    * 1) Shutdown due to inactivity. RunThread() detects that it has no
    * pending messages and no streams, and exits. The next RunInStableState()
    * checks if there are new pending messages from the main thread (true only
    * if new stream creation raced with shutdown); if there are, it revives
    * RunThread(), otherwise it commits to shutting down the graph. New stream
    * creation after this point will create a new graph. An async event is
    * dispatched to Shutdown() the graph's threads and then delete the graph
    * object.
    * 2) Forced shutdown at application shutdown, or completion of a
    * non-realtime graph. A flag is set, RunThread() detects the flag and
    * exits, the next RunInStableState() detects the flag, and dispatches the
    * async event to Shutdown() the graph's threads. However the graph object
    * is not deleted. New messages for the graph are processed synchronously on
    * the main thread if necessary. When the last stream is destroyed, the
    * graph object is deleted.
    */
   enum LifecycleState {
     // The graph thread hasn't started yet.
     LIFECYCLE_THREAD_NOT_STARTED,
     // RunThread() is running normally.
     LIFECYCLE_RUNNING,
     // In the following states, the graph thread is not running so
     // all "graph thread only" state in this class can be used safely
     // on the main thread.
     // RunThread() has exited and we're waiting for the next
     // RunInStableState(), at which point we can clean up the main-thread
     // side of the graph.
     LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP,
     // RunInStableState() posted a ShutdownRunnable, and we're waiting for it
     // to shut down the graph thread(s).
     LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN,
     // Graph threads have shut down but we're waiting for remaining streams
     // to be destroyed. Only happens during application shutdown and on
     // completed non-realtime graphs, since normally we'd only shut down a
     // realtime graph when it has no streams.
     LIFECYCLE_WAITING_FOR_STREAM_DESTRUCTION
   };
   LifecycleState mLifecycleState;
   /**
    * This enum specifies the wait state of the graph thread.
    */
   enum WaitState {
     // RunThread() is running normally
     WAITSTATE_RUNNING,
     // RunThread() is paused waiting for its next iteration, which will
     // happen soon
     WAITSTATE_WAITING_FOR_NEXT_ITERATION,
     // RunThread() is paused indefinitely waiting for something to change
     WAITSTATE_WAITING_INDEFINITELY,
     // Something has signaled RunThread() to wake up immediately,
     // but it hasn't done so yet
     WAITSTATE_WAKING_UP
   };
   WaitState mWaitState;
   /**
    * The graph should stop processing at or after this time.
    */
   GraphTime mEndTime;
   /**
    * Sample rate at which this graph runs. For real time graphs, this is
    * the rate of the audio mixer. For offline graphs, this is the rate specified
    * at construction.
    */
   TrackRate mSampleRate;
   /**
    * True when another iteration of the control loop is required.
    */
   bool mNeedAnotherIteration;
   /**
    * True when we need to do a forced shutdown during application shutdown.
    */
   bool mForceShutDown;
   /**
    * True when we have posted an event to the main thread to run
    * RunInStableState() and the event hasn't run yet.
    */
   bool mPostedRunInStableStateEvent;
   // Main thread only
   /**
    * Messages posted by the current event loop task. These are forwarded to
    * the media graph thread during RunInStableState. We can't forward them
    * immediately because we want all messages between stable states to be
    * processed as an atomic batch.
    */
   nsTArray<nsAutoPtr<ControlMessage> > mCurrentTaskMessageQueue;
   /**
    * True when RunInStableState has determined that mLifecycleState is >
    * LIFECYCLE_RUNNING. Since only the main thread can reset mLifecycleState to
    * LIFECYCLE_RUNNING, this can be relied on to not change unexpectedly.
    */
   bool mDetectedNotRunning;
   /**
    * True when a stable state runner has been posted to the appshell to run
    * RunInStableState at the next stable state.
    */
   bool mPostedRunInStableState;
   /**
    * True when processing real-time audio/video.  False when processing non-realtime
    * audio.
    */
   bool mRealtime;
   /**
    * True when a non-realtime MediaStreamGraph has started to process input.  This
    * value is only accessed on the main thread.
    */
   bool mNonRealtimeProcessing;
   /**
    * True when a change has happened which requires us to recompute the stream
    * blocking order.
    */
   bool mStreamOrderDirty;
   /**
    * Hold a ref to the Latency logger
    */
   nsRefPtr<AsyncLatencyLogger> mLatencyLog;
   /**
    * If this is not null, all the audio output for the MSG will be mixed down.
    */
   nsAutoPtr<AudioMixer> mMixer;
 private:
   virtual ~MediaStreamGraphImpl();
   MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf)
   /**
    * Used to signal that a memory report has been requested.
    */
   Monitor mMemoryReportMonitor;
   /**
    * This class uses manual memory management, and all pointers to it are raw
    * pointers. However, in order for it to implement nsIMemoryReporter, it needs
    * to implement nsISupports and so be ref-counted. So it maintains a single
    * nsRefPtr to itself, giving it a ref-count of 1 during its entire lifetime,
    * and Destroy() nulls this self-reference in order to trigger self-deletion.
    */
   nsRefPtr<MediaStreamGraphImpl> mSelfRef;
   /**
    * Used to pass memory report information across threads.
    */
   nsTArray<AudioNodeSizes> mAudioStreamSizes;
   /**
    * Indicates that the MSG thread should gather data for a memory report.
    */
   bool mNeedsMemoryReport;
 #ifdef DEBUG
   /**
    * Used to assert when AppendMessage() runs ControlMessages synchronously.
    */
   bool mCanRunMessagesSynchronously;
 #endif
 };
 }
 #endif /* MEDIASTREAMGRAPHIMPL_H_ */

The Tor Browser / annotate

content/media/MediaStreamGraphImpl.h@97036ab72558 (annotated)

content/media/MediaStreamGraphImpl.h