The Tor Browser: comparison content/media/MediaStreamGraph.h

--1:000000000000
+:c0a99c19fae1
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
+/* This Source Code Form is subject to the terms of the Mozilla Public
+* License, v. 2.0. If a copy of the MPL was not distributed with this file,
+* You can obtain one at http://mozilla.org/MPL/2.0/. */
+#ifndef MOZILLA_MEDIASTREAMGRAPH_H_
+#define MOZILLA_MEDIASTREAMGRAPH_H_
+#include "mozilla/Mutex.h"
+#include "mozilla/LinkedList.h"
+#include "AudioStream.h"
+#include "nsTArray.h"
+#include "nsIRunnable.h"
+#include "StreamBuffer.h"
+#include "TimeVarying.h"
+#include "VideoFrameContainer.h"
+#include "VideoSegment.h"
+#include "MainThreadUtils.h"
+#include "nsAutoRef.h"
+#include "speex/speex_resampler.h"
+#include "AudioMixer.h"
+#include "mozilla/dom/AudioChannelBinding.h"
+class nsIRunnable;
+template <>
+class nsAutoRefTraits<SpeexResamplerState> : public nsPointerRefTraits<SpeexResamplerState>
+{
+public:
+static void Release(SpeexResamplerState* aState) { speex_resampler_destroy(aState); }
+};
+namespace mozilla {
+class DOMMediaStream;
+#ifdef PR_LOGGING
+extern PRLogModuleInfo* gMediaStreamGraphLog;
+#endif
+/**
+* Microseconds relative to the start of the graph timeline.
+*/
+typedef int64_t GraphTime;
+const GraphTime GRAPH_TIME_MAX = MEDIA_TIME_MAX;
+/*
+* MediaStreamGraph is a framework for synchronized audio/video processing
+* and playback. It is designed to be used by other browser components such as
+* HTML media elements, media capture APIs, real-time media streaming APIs,
+* multitrack media APIs, and advanced audio APIs.
+*
+* The MediaStreamGraph uses a dedicated thread to process media --- the media
+* graph thread. This ensures that we can process media through the graph
+* without blocking on main-thread activity. The media graph is only modified
+* on the media graph thread, to ensure graph changes can be processed without
+* interfering with media processing. All interaction with the media graph
+* thread is done with message passing.
+*
+* APIs that modify the graph or its properties are described as "control APIs".
+* These APIs are asynchronous; they queue graph changes internally and
+* those changes are processed all-at-once by the MediaStreamGraph. The
+* MediaStreamGraph monitors the main thread event loop via nsIAppShell::RunInStableState
+* to ensure that graph changes from a single event loop task are always
+* processed all together. Control APIs should only be used on the main thread,
+* currently; we may be able to relax that later.
+*
+* To allow precise synchronization of times in the control API, the
+* MediaStreamGraph maintains a "media timeline". Control APIs that take or
+* return times use that timeline. Those times never advance during
+* an event loop task. This time is returned by MediaStreamGraph::GetCurrentTime().
+*
+* Media decoding, audio processing and media playback use thread-safe APIs to
+* the media graph to ensure they can continue while the main thread is blocked.
+*
+* When the graph is changed, we may need to throw out buffered data and
+* reprocess it. This is triggered automatically by the MediaStreamGraph.
+*/
+class MediaStreamGraph;
+/**
+* This is a base class for media graph thread listener callbacks.
+* Override methods to be notified of audio or video data or changes in stream
+* state.
+*
+* This can be used by stream recorders or network connections that receive
+* stream input. It could also be used for debugging.
+*
+* All notification methods are called from the media graph thread. Overriders
+* of these methods are responsible for all synchronization. Beware!
+* These methods are called without the media graph monitor held, so
+* reentry into media graph methods is possible, although very much discouraged!
+* You should do something non-blocking and non-reentrant (e.g. dispatch an
+* event to some thread) and return.
+* The listener is not allowed to add/remove any listeners from the stream.
+*
+* When a listener is first attached, we guarantee to send a NotifyBlockingChanged
+* callback to notify of the initial blocking state. Also, if a listener is
+* attached to a stream that has already finished, we'll call NotifyFinished.
+*/
+class MediaStreamListener {
+protected:
+// Protected destructor, to discourage deletion outside of Release():
+virtual ~MediaStreamListener() {}
+public:
+NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStreamListener)
+enum Consumption {
+CONSUMED,
+NOT_CONSUMED
+};
+/**
+* Notify that the stream is hooked up and we'd like to start or stop receiving
+* data on it. Only fires on SourceMediaStreams.
+* The initial state is assumed to be NOT_CONSUMED.
+*/
+virtual void NotifyConsumptionChanged(MediaStreamGraph* aGraph, Consumption aConsuming) {}
+/**
+* When a SourceMediaStream has pulling enabled, and the MediaStreamGraph
+* control loop is ready to pull, this gets called. A NotifyPull implementation
+* is allowed to call the SourceMediaStream methods that alter track
+* data. It is not allowed to make other MediaStream API calls, including
+* calls to add or remove MediaStreamListeners. It is not allowed to block
+* for any length of time.
+* aDesiredTime is the stream time we would like to get data up to. Data
+* beyond this point will not be played until NotifyPull runs again, so there's
+* not much point in providing it. Note that if the stream is blocked for
+* some reason, then data before aDesiredTime may not be played immediately.
+*/
+virtual void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime) {}
+enum Blocking {
+BLOCKED,
+UNBLOCKED
+};
+/**
+* Notify that the blocking status of the stream changed. The initial state
+* is assumed to be BLOCKED.
+*/
+virtual void NotifyBlockingChanged(MediaStreamGraph* aGraph, Blocking aBlocked) {}
+/**
+* Notify that the stream has data in each track
+* for the stream's current time. Once this state becomes true, it will
+* always be true since we block stream time from progressing to times where
+* there isn't data in each track.
+*/
+virtual void NotifyHasCurrentData(MediaStreamGraph* aGraph) {}
+/**
+* Notify that the stream output is advancing. aCurrentTime is the graph's
+* current time. MediaStream::GraphTimeToStreamTime can be used to get the
+* stream time.
+*/
+virtual void NotifyOutput(MediaStreamGraph* aGraph, GraphTime aCurrentTime) {}
+/**
+* Notify that the stream finished.
+*/
+virtual void NotifyFinished(MediaStreamGraph* aGraph) {}
+/**
+* Notify that your listener has been removed, either due to RemoveListener(),
+* or due to the stream being destroyed.  You will get no further notifications.
+*/
+virtual void NotifyRemoved(MediaStreamGraph* aGraph) {}
+enum {
+TRACK_EVENT_CREATED = 0x01,
+TRACK_EVENT_ENDED = 0x02
+};
+/**
+* Notify that changes to one of the stream tracks have been queued.
+* aTrackEvents can be any combination of TRACK_EVENT_CREATED and
+* TRACK_EVENT_ENDED. aQueuedMedia is the data being added to the track
+* at aTrackOffset (relative to the start of the stream).
+*/
+virtual void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
+TrackRate aTrackRate,
+TrackTicks aTrackOffset,
+uint32_t aTrackEvents,
+const MediaSegment& aQueuedMedia) {}
+};
+/**
+* This is a base class for media graph thread listener direct callbacks
+* from within AppendToTrack().  Note that your regular listener will
+* still get NotifyQueuedTrackChanges() callbacks from the MSG thread, so
+* you must be careful to ignore them if AddDirectListener was successful.
+*/
+class MediaStreamDirectListener : public MediaStreamListener
+{
+public:
+virtual ~MediaStreamDirectListener() {}
+/*
+* This will be called on any MediaStreamDirectListener added to a
+* a SourceMediaStream when AppendToTrack() is called.  The MediaSegment
+* will be the RawSegment (unresampled) if available in AppendToTrack().
+* Note that NotifyQueuedTrackChanges() calls will also still occur.
+*/
+virtual void NotifyRealtimeData(MediaStreamGraph* aGraph, TrackID aID,
+TrackRate aTrackRate,
+TrackTicks aTrackOffset,
+uint32_t aTrackEvents,
+const MediaSegment& aMedia) {}
+};
+/**
+* This is a base class for main-thread listener callbacks.
+* This callback is invoked on the main thread when the main-thread-visible
+* state of a stream has changed.
+*
+* These methods are called with the media graph monitor held, so
+* reentry into general media graph methods is not possible.
+* You should do something non-blocking and non-reentrant (e.g. dispatch an
+* event) and return. DispatchFromMainThreadAfterNextStreamStateUpdate
+* would be a good choice.
+* The listener is allowed to synchronously remove itself from the stream, but
+* not add or remove any other listeners.
+*/
+class MainThreadMediaStreamListener {
+public:
+virtual void NotifyMainThreadStateChanged() = 0;
+};
+/**
+* Helper struct used to keep track of memory usage by AudioNodes.
+*/
+struct AudioNodeSizes
+{
+size_t mDomNode;
+size_t mStream;
+size_t mEngine;
+nsCString mNodeType;
+};
+class MediaStreamGraphImpl;
+class SourceMediaStream;
+class ProcessedMediaStream;
+class MediaInputPort;
+class AudioNodeEngine;
+class AudioNodeExternalInputStream;
+class AudioNodeStream;
+struct AudioChunk;
+/**
+* A stream of synchronized audio and video data. All (not blocked) streams
+* progress at the same rate --- "real time". Streams cannot seek. The only
+* operation readers can perform on a stream is to read the next data.
+*
+* Consumers of a stream can be reading from it at different offsets, but that
+* should only happen due to the order in which consumers are being run.
+* Those offsets must not diverge in the long term, otherwise we would require
+* unbounded buffering.
+*
+* Streams can be in a "blocked" state. While blocked, a stream does not
+* produce data. A stream can be explicitly blocked via the control API,
+* or implicitly blocked by whatever's generating it (e.g. an underrun in the
+* source resource), or implicitly blocked because something consuming it
+* blocks, or implicitly because it has finished.
+*
+* A stream can be in a "finished" state. "Finished" streams are permanently
+* blocked.
+*
+* Transitions into and out of the "blocked" and "finished" states are managed
+* by the MediaStreamGraph on the media graph thread.
+*
+* We buffer media data ahead of the consumers' reading offsets. It is possible
+* to have buffered data but still be blocked.
+*
+* Any stream can have its audio and video playing when requested. The media
+* stream graph plays audio by constructing audio output streams as necessary.
+* Video is played by setting video frames into an VideoFrameContainer at the right
+* time. To ensure video plays in sync with audio, make sure that the same
+* stream is playing both the audio and video.
+*
+* The data in a stream is managed by StreamBuffer. It consists of a set of
+* tracks of various types that can start and end over time.
+*
+* Streams are explicitly managed. The client creates them via
+* MediaStreamGraph::CreateInput/ProcessedMediaStream, and releases them by calling
+* Destroy() when no longer needed (actual destruction will be deferred).
+* The actual object is owned by the MediaStreamGraph. The basic idea is that
+* main thread objects will keep Streams alive as long as necessary (using the
+* cycle collector to clean up whenever needed).
+*
+* We make them refcounted only so that stream-related messages with MediaStream*
+* pointers can be sent to the main thread safely.
+*
+* The lifetimes of MediaStreams are controlled from the main thread.
+* For MediaStreams exposed to the DOM, the lifetime is controlled by the DOM
+* wrapper; the DOM wrappers own their associated MediaStreams. When a DOM
+* wrapper is destroyed, it sends a Destroy message for the associated
+* MediaStream and clears its reference (the last main-thread reference to
+* the object). When the Destroy message is processed on the graph
+* manager thread we immediately release the affected objects (disentangling them
+* from other objects as necessary).
+*
+* This could cause problems for media processing if a MediaStream is
+* destroyed while a downstream MediaStream is still using it. Therefore
+* the DOM wrappers must keep upstream MediaStreams alive as long as they
+* could be being used in the media graph.
+*
+* At any time, however, a set of MediaStream wrappers could be
+* collected via cycle collection. Destroy messages will be sent
+* for those objects in arbitrary order and the MediaStreamGraph has to be able
+* to handle this.
+*/
+class MediaStream : public mozilla::LinkedListElement<MediaStream> {
+public:
+NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStream)
+MediaStream(DOMMediaStream* aWrapper);
+protected:
+// Protected destructor, to discourage deletion outside of Release():
+virtual ~MediaStream()
+{
+MOZ_COUNT_DTOR(MediaStream);
+NS_ASSERTION(mMainThreadDestroyed, "Should have been destroyed already");
+NS_ASSERTION(mMainThreadListeners.IsEmpty(),
+"All main thread listeners should have been removed");
+}
+public:
+/**
+* Returns the graph that owns this stream.
+*/
+MediaStreamGraphImpl* GraphImpl();
+MediaStreamGraph* Graph();
+/**
+* Sets the graph that owns this stream.  Should only be called once.
+*/
+void SetGraphImpl(MediaStreamGraphImpl* aGraph);
+void SetGraphImpl(MediaStreamGraph* aGraph);
+// Control API.
+// Since a stream can be played multiple ways, we need to combine independent
+// volume settings. The aKey parameter is used to keep volume settings
+// separate. Since the stream is always playing the same contents, only
+// a single audio output stream is used; the volumes are combined.
+// Currently only the first enabled audio track is played.
+// XXX change this so all enabled audio tracks are mixed and played.
+virtual void AddAudioOutput(void* aKey);
+virtual void SetAudioOutputVolume(void* aKey, float aVolume);
+virtual void RemoveAudioOutput(void* aKey);
+// Since a stream can be played multiple ways, we need to be able to
+// play to multiple VideoFrameContainers.
+// Only the first enabled video track is played.
+virtual void AddVideoOutput(VideoFrameContainer* aContainer);
+virtual void RemoveVideoOutput(VideoFrameContainer* aContainer);
+// Explicitly block. Useful for example if a media element is pausing
+// and we need to stop its stream emitting its buffered data.
+virtual void ChangeExplicitBlockerCount(int32_t aDelta);
+// Events will be dispatched by calling methods of aListener.
+virtual void AddListener(MediaStreamListener* aListener);
+virtual void RemoveListener(MediaStreamListener* aListener);
+// A disabled track has video replaced by black, and audio replaced by
+// silence.
+void SetTrackEnabled(TrackID aTrackID, bool aEnabled);
+// Events will be dispatched by calling methods of aListener. It is the
+// responsibility of the caller to remove aListener before it is destroyed.
+void AddMainThreadListener(MainThreadMediaStreamListener* aListener)
+{
+NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
+mMainThreadListeners.AppendElement(aListener);
+}
+// It's safe to call this even if aListener is not currently a listener;
+// the call will be ignored.
+void RemoveMainThreadListener(MainThreadMediaStreamListener* aListener)
+{
+NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
+mMainThreadListeners.RemoveElement(aListener);
+}
+/**
+* Ensure a runnable will run on the main thread after running all pending
+* updates that were sent from the graph thread or will be sent before the
+* graph thread receives the next graph update.
+*
+* If the graph has been shut down or destroyed, then the runnable will be
+* dispatched to the event queue immediately.  If the graph is non-realtime
+* and has not started, then the runnable will be run
+* synchronously/immediately.  (There are no pending updates in these
+* situations.)
+*
+* Main thread only.
+*/
+void RunAfterPendingUpdates(nsRefPtr<nsIRunnable> aRunnable);
+// Signal that the client is done with this MediaStream. It will be deleted later.
+virtual void Destroy();
+// Returns the main-thread's view of how much data has been processed by
+// this stream.
+StreamTime GetCurrentTime()
+{
+NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
+return mMainThreadCurrentTime;
+}
+// Return the main thread's view of whether this stream has finished.
+bool IsFinished()
+{
+NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
+return mMainThreadFinished;
+}
+bool IsDestroyed()
+{
+NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
+return mMainThreadDestroyed;
+}
+friend class MediaStreamGraphImpl;
+friend class MediaInputPort;
+friend class AudioNodeExternalInputStream;
+virtual SourceMediaStream* AsSourceStream() { return nullptr; }
+virtual ProcessedMediaStream* AsProcessedStream() { return nullptr; }
+virtual AudioNodeStream* AsAudioNodeStream() { return nullptr; }
+// media graph thread only
+void Init();
+// These Impl methods perform the core functionality of the control methods
+// above, on the media graph thread.
+/**
+* Stop all stream activity and disconnect it from all inputs and outputs.
+* This must be idempotent.
+*/
+virtual void DestroyImpl();
+StreamTime GetBufferEnd() { return mBuffer.GetEnd(); }
+#ifdef DEBUG
+void DumpTrackInfo() { return mBuffer.DumpTrackInfo(); }
+#endif
+void SetAudioOutputVolumeImpl(void* aKey, float aVolume);
+void AddAudioOutputImpl(void* aKey)
+{
+mAudioOutputs.AppendElement(AudioOutput(aKey));
+}
+void RemoveAudioOutputImpl(void* aKey);
+void AddVideoOutputImpl(already_AddRefed<VideoFrameContainer> aContainer)
+{
+*mVideoOutputs.AppendElement() = aContainer;
+}
+void RemoveVideoOutputImpl(VideoFrameContainer* aContainer)
+{
+mVideoOutputs.RemoveElement(aContainer);
+}
+void ChangeExplicitBlockerCountImpl(GraphTime aTime, int32_t aDelta)
+{
+mExplicitBlockerCount.SetAtAndAfter(aTime, mExplicitBlockerCount.GetAt(aTime) + aDelta);
+}
+void AddListenerImpl(already_AddRefed<MediaStreamListener> aListener);
+void RemoveListenerImpl(MediaStreamListener* aListener);
+void RemoveAllListenersImpl();
+void SetTrackEnabledImpl(TrackID aTrackID, bool aEnabled);
+/**
+* Returns true when this stream requires the contents of its inputs even if
+* its own outputs are not being consumed. This is used to signal inputs to
+* this stream that they are being consumed; when they're not being consumed,
+* we make some optimizations.
+*/
+virtual bool IsIntrinsicallyConsumed() const
+{
+return !mAudioOutputs.IsEmpty() || !mVideoOutputs.IsEmpty();
+}
+void AddConsumer(MediaInputPort* aPort)
+{
+mConsumers.AppendElement(aPort);
+}
+void RemoveConsumer(MediaInputPort* aPort)
+{
+mConsumers.RemoveElement(aPort);
+}
+uint32_t ConsumerCount()
+{
+return mConsumers.Length();
+}
+const StreamBuffer& GetStreamBuffer() { return mBuffer; }
+GraphTime GetStreamBufferStartTime() { return mBufferStartTime; }
+/**
+* Convert graph time to stream time. aTime must be <= mStateComputedTime
+* to ensure we know exactly how much time this stream will be blocked during
+* the interval.
+*/
+StreamTime GraphTimeToStreamTime(GraphTime aTime);
+/**
+* Convert graph time to stream time. aTime can be > mStateComputedTime,
+* in which case we optimistically assume the stream will not be blocked
+* after mStateComputedTime.
+*/
+StreamTime GraphTimeToStreamTimeOptimistic(GraphTime aTime);
+/**
+* Convert stream time to graph time. The result can be > mStateComputedTime,
+* in which case we did the conversion optimistically assuming the stream
+* will not be blocked after mStateComputedTime.
+*/
+GraphTime StreamTimeToGraphTime(StreamTime aTime);
+bool IsFinishedOnGraphThread() { return mFinished; }
+void FinishOnGraphThread();
+/**
+* Identify which graph update index we are currently processing.
+*/
+int64_t GetProcessingGraphUpdateIndex();
+bool HasCurrentData() { return mHasCurrentData; }
+StreamBuffer::Track* EnsureTrack(TrackID aTrack, TrackRate aSampleRate);
+void ApplyTrackDisabling(TrackID aTrackID, MediaSegment* aSegment, MediaSegment* aRawSegment = nullptr);
+DOMMediaStream* GetWrapper()
+{
+NS_ASSERTION(NS_IsMainThread(), "Only use DOMMediaStream on main thread");
+return mWrapper;
+}
+// Return true if the main thread needs to observe updates from this stream.
+virtual bool MainThreadNeedsUpdates() const
+{
+return true;
+}
+virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const;
+virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const;
+void SetAudioChannelType(dom::AudioChannel aType) { mAudioChannelType = aType; }
+protected:
+virtual void AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime, GraphTime aBlockedTime)
+{
+mBufferStartTime += aBlockedTime;
+mGraphUpdateIndices.InsertTimeAtStart(aBlockedTime);
+mGraphUpdateIndices.AdvanceCurrentTime(aCurrentTime);
+mExplicitBlockerCount.AdvanceCurrentTime(aCurrentTime);
+mBuffer.ForgetUpTo(aCurrentTime - mBufferStartTime);
+}
+// This state is all initialized on the main thread but
+// otherwise modified only on the media graph thread.
+// Buffered data. The start of the buffer corresponds to mBufferStartTime.
+// Conceptually the buffer contains everything this stream has ever played,
+// but we forget some prefix of the buffered data to bound the space usage.
+StreamBuffer mBuffer;
+// The time when the buffered data could be considered to have started playing.
+// This increases over time to account for time the stream was blocked before
+// mCurrentTime.
+GraphTime mBufferStartTime;
+// Client-set volume of this stream
+struct AudioOutput {
+AudioOutput(void* aKey) : mKey(aKey), mVolume(1.0f) {}
+void* mKey;
+float mVolume;
+};
+nsTArray<AudioOutput> mAudioOutputs;
+nsTArray<nsRefPtr<VideoFrameContainer> > mVideoOutputs;
+// We record the last played video frame to avoid redundant setting
+// of the current video frame.
+VideoFrame mLastPlayedVideoFrame;
+// The number of times this stream has been explicitly blocked by the control
+// API, minus the number of times it has been explicitly unblocked.
+TimeVarying<GraphTime,uint32_t,0> mExplicitBlockerCount;
+nsTArray<nsRefPtr<MediaStreamListener> > mListeners;
+nsTArray<MainThreadMediaStreamListener*> mMainThreadListeners;
+nsTArray<TrackID> mDisabledTrackIDs;
+// Precomputed blocking status (over GraphTime).
+// This is only valid between the graph's mCurrentTime and
+// mStateComputedTime. The stream is considered to have
+// not been blocked before mCurrentTime (its mBufferStartTime is increased
+// as necessary to account for that time instead) --- this avoids us having to
+// record the entire history of the stream's blocking-ness in mBlocked.
+TimeVarying<GraphTime,bool,5> mBlocked;
+// Maps graph time to the graph update that affected this stream at that time
+TimeVarying<GraphTime,int64_t,0> mGraphUpdateIndices;
+// MediaInputPorts to which this is connected
+nsTArray<MediaInputPort*> mConsumers;
+// Where audio output is going. There is one AudioOutputStream per
+// audio track.
+struct AudioOutputStream {
+// When we started audio playback for this track.
+// Add mStream->GetPosition() to find the current audio playback position.
+GraphTime mAudioPlaybackStartTime;
+// Amount of time that we've wanted to play silence because of the stream
+// blocking.
+MediaTime mBlockedAudioTime;
+// Last tick written to the audio output.
+TrackTicks mLastTickWritten;
+RefPtr<AudioStream> mStream;
+TrackID mTrackID;
+size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const
+{
+size_t amount = 0;
+amount += mStream->SizeOfIncludingThis(aMallocSizeOf);
+return amount;
+}
+};
+nsTArray<AudioOutputStream> mAudioOutputStreams;
+/**
+* When true, this means the stream will be finished once all
+* buffered data has been consumed.
+*/
+bool mFinished;
+/**
+* When true, mFinished is true and we've played all the data in this stream
+* and fired NotifyFinished notifications.
+*/
+bool mNotifiedFinished;
+/**
+* When true, the last NotifyBlockingChanged delivered to the listeners
+* indicated that the stream is blocked.
+*/
+bool mNotifiedBlocked;
+/**
+* True if some data can be present by this stream if/when it's unblocked.
+* Set by the stream itself on the MediaStreamGraph thread. Only changes
+* from false to true once a stream has data, since we won't
+* unblock it until there's more data.
+*/
+bool mHasCurrentData;
+/**
+* True if mHasCurrentData is true and we've notified listeners.
+*/
+bool mNotifiedHasCurrentData;
+// Temporary data for ordering streams by dependency graph
+bool mHasBeenOrdered;
+bool mIsOnOrderingStack;
+// True if the stream is being consumed (i.e. has track data being played,
+// or is feeding into some stream that is being consumed).
+bool mIsConsumed;
+// Temporary data for computing blocking status of streams
+// True if we've added this stream to the set of streams we're computing
+// blocking for.
+bool mInBlockingSet;
+// True if this stream should be blocked in this phase.
+bool mBlockInThisPhase;
+// This state is only used on the main thread.
+DOMMediaStream* mWrapper;
+// Main-thread views of state
+StreamTime mMainThreadCurrentTime;
+bool mMainThreadFinished;
+bool mMainThreadDestroyed;
+// Our media stream graph.  null if destroyed on the graph thread.
+MediaStreamGraphImpl* mGraph;
+dom::AudioChannel mAudioChannelType;
+};
+/**
+* This is a stream into which a decoder can write audio and video.
+*
+* Audio and video can be written on any thread, but you probably want to
+* always write from the same thread to avoid unexpected interleavings.
+*/
+class SourceMediaStream : public MediaStream {
+public:
+SourceMediaStream(DOMMediaStream* aWrapper) :
+MediaStream(aWrapper),
+mLastConsumptionState(MediaStreamListener::NOT_CONSUMED),
+mMutex("mozilla::media::SourceMediaStream"),
+mUpdateKnownTracksTime(0),
+mPullEnabled(false),
+mUpdateFinished(false)
+{}
+virtual SourceMediaStream* AsSourceStream() { return this; }
+// Media graph thread only
+virtual void DestroyImpl();
+// Call these on any thread.
+/**
+* Enable or disable pulling. When pulling is enabled, NotifyPull
+* gets called on MediaStreamListeners for this stream during the
+* MediaStreamGraph control loop. Pulling is initially disabled.
+* Due to unavoidable race conditions, after a call to SetPullEnabled(false)
+* it is still possible for a NotifyPull to occur.
+*/
+void SetPullEnabled(bool aEnabled);
+void AddDirectListener(MediaStreamDirectListener* aListener);
+void RemoveDirectListener(MediaStreamDirectListener* aListener);
+/**
+* Add a new track to the stream starting at the given base time (which
+* must be greater than or equal to the last time passed to
+* AdvanceKnownTracksTime). Takes ownership of aSegment. aSegment should
+* contain data starting after aStart.
+*/
+void AddTrack(TrackID aID, TrackRate aRate, TrackTicks aStart,
+MediaSegment* aSegment);
+/**
+* Append media data to a track. Ownership of aSegment remains with the caller,
+* but aSegment is emptied.
+* Returns false if the data was not appended because no such track exists
+* or the stream was already finished.
+*/
+bool AppendToTrack(TrackID aID, MediaSegment* aSegment, MediaSegment *aRawSegment = nullptr);
+/**
+* Returns true if the buffer currently has enough data.
+* Returns false if there isn't enough data or if no such track exists.
+*/
+bool HaveEnoughBuffered(TrackID aID);
+/**
+* Ensures that aSignalRunnable will be dispatched to aSignalThread
+* when we don't have enough buffered data in the track (which could be
+* immediately). Will dispatch the runnable immediately if the track
+* does not exist. No op if a runnable is already present for this track.
+*/
+void DispatchWhenNotEnoughBuffered(TrackID aID,
+nsIEventTarget* aSignalThread, nsIRunnable* aSignalRunnable);
+/**
+* Indicate that a track has ended. Do not do any more API calls
+* affecting this track.
+* Ignored if the track does not exist.
+*/
+void EndTrack(TrackID aID);
+/**
+* Indicate that no tracks will be added starting before time aKnownTime.
+* aKnownTime must be >= its value at the last call to AdvanceKnownTracksTime.
+*/
+void AdvanceKnownTracksTime(StreamTime aKnownTime);
+/**
+* Indicate that this stream should enter the "finished" state. All tracks
+* must have been ended via EndTrack. The finish time of the stream is
+* when all tracks have ended.
+*/
+void FinishWithLockHeld();
+void Finish()
+{
+MutexAutoLock lock(mMutex);
+FinishWithLockHeld();
+}
+// Overriding allows us to hold the mMutex lock while changing the track enable status
+void SetTrackEnabledImpl(TrackID aTrackID, bool aEnabled) {
+MutexAutoLock lock(mMutex);
+MediaStream::SetTrackEnabledImpl(aTrackID, aEnabled);
+}
+/**
+* End all tracks and Finish() this stream.  Used to voluntarily revoke access
+* to a LocalMediaStream.
+*/
+void EndAllTrackAndFinish();
+/**
+* Note: Only call from Media Graph thread (eg NotifyPull)
+*
+* Returns amount of time (data) that is currently buffered in the track,
+* assuming playout via PlayAudio or via a TrackUnion - note that
+* NotifyQueuedTrackChanges() on a SourceMediaStream will occur without
+* any "extra" buffering, but NotifyQueued TrackChanges() on a TrackUnion
+* will be buffered.
+*/
+TrackTicks GetBufferedTicks(TrackID aID);
+void RegisterForAudioMixing();
+// XXX need a Reset API
+friend class MediaStreamGraphImpl;
+protected:
+struct ThreadAndRunnable {
+void Init(nsIEventTarget* aTarget, nsIRunnable* aRunnable)
+{
+mTarget = aTarget;
+mRunnable = aRunnable;
+}
+nsCOMPtr<nsIEventTarget> mTarget;
+nsCOMPtr<nsIRunnable> mRunnable;
+};
+enum TrackCommands {
+TRACK_CREATE = MediaStreamListener::TRACK_EVENT_CREATED,
+TRACK_END = MediaStreamListener::TRACK_EVENT_ENDED
+};
+/**
+* Data for each track that hasn't ended.
+*/
+struct TrackData {
+TrackID mID;
+// Sample rate of the input data.
+TrackRate mInputRate;
+// Sample rate of the output data, always equal to the sample rate of the
+// graph.
+TrackRate mOutputRate;
+// Resampler if the rate of the input track does not match the
+// MediaStreamGraph's.
+nsAutoRef<SpeexResamplerState> mResampler;
+TrackTicks mStart;
+// Each time the track updates are flushed to the media graph thread,
+// this is cleared.
+uint32_t mCommands;
+// Each time the track updates are flushed to the media graph thread,
+// the segment buffer is emptied.
+nsAutoPtr<MediaSegment> mData;
+nsTArray<ThreadAndRunnable> mDispatchWhenNotEnough;
+bool mHaveEnough;
+};
+bool NeedsMixing();
+void ResampleAudioToGraphSampleRate(TrackData* aTrackData, MediaSegment* aSegment);
+TrackData* FindDataForTrack(TrackID aID)
+{
+for (uint32_t i = 0; i < mUpdateTracks.Length(); ++i) {
+if (mUpdateTracks[i].mID == aID) {
+return &mUpdateTracks[i];
+}
+}
+return nullptr;
+}
+/**
+* Notify direct consumers of new data to one of the stream tracks.
+* The data doesn't have to be resampled (though it may be).  This is called
+* from AppendToTrack on the thread providing the data, and will call
+* the Listeners on this thread.
+*/
+void NotifyDirectConsumers(TrackData *aTrack,
+MediaSegment *aSegment);
+// Media stream graph thread only
+MediaStreamListener::Consumption mLastConsumptionState;
+// This must be acquired *before* MediaStreamGraphImpl's lock, if they are
+// held together.
+Mutex mMutex;
+// protected by mMutex
+StreamTime mUpdateKnownTracksTime;
+nsTArray<TrackData> mUpdateTracks;
+nsTArray<nsRefPtr<MediaStreamDirectListener> > mDirectListeners;
+bool mPullEnabled;
+bool mUpdateFinished;
+bool mNeedsMixing;
+};
+/**
+* Represents a connection between a ProcessedMediaStream and one of its
+* input streams.
+* We make these refcounted so that stream-related messages with MediaInputPort*
+* pointers can be sent to the main thread safely.
+*
+* When a port's source or destination stream dies, the stream's DestroyImpl
+* calls MediaInputPort::Disconnect to disconnect the port from
+* the source and destination streams.
+*
+* The lifetimes of MediaInputPort are controlled from the main thread.
+* The media graph adds a reference to the port. When a MediaInputPort is no
+* longer needed, main-thread code sends a Destroy message for the port and
+* clears its reference (the last main-thread reference to the object). When
+* the Destroy message is processed on the graph manager thread we disconnect
+* the port and drop the graph's reference, destroying the object.
+*/
+class MediaInputPort MOZ_FINAL {
+private:
+// Do not call this constructor directly. Instead call aDest->AllocateInputPort.
+MediaInputPort(MediaStream* aSource, ProcessedMediaStream* aDest,
+uint32_t aFlags, uint16_t aInputNumber,
+uint16_t aOutputNumber)
+: mSource(aSource)
+, mDest(aDest)
+, mFlags(aFlags)
+, mInputNumber(aInputNumber)
+, mOutputNumber(aOutputNumber)
+, mGraph(nullptr)
+{
+MOZ_COUNT_CTOR(MediaInputPort);
+}
+// Private destructor, to discourage deletion outside of Release():
+~MediaInputPort()
+{
+MOZ_COUNT_DTOR(MediaInputPort);
+}
+public:
+NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaInputPort)
+/**
+* The FLAG_BLOCK_INPUT and FLAG_BLOCK_OUTPUT flags can be used to control
+* exactly how the blocking statuses of the input and output streams affect
+* each other.
+*/
+enum {
+// When set, blocking on the output stream forces blocking on the input
+// stream.
+FLAG_BLOCK_INPUT = 0x01,
+// When set, blocking on the input stream forces blocking on the output
+// stream.
+FLAG_BLOCK_OUTPUT = 0x02
+};
+// Called on graph manager thread
+// Do not call these from outside MediaStreamGraph.cpp!
+void Init();
+// Called during message processing to trigger removal of this stream.
+void Disconnect();
+// Control API
+/**
+* Disconnects and destroys the port. The caller must not reference this
+* object again.
+*/
+void Destroy();
+// Any thread
+MediaStream* GetSource() { return mSource; }
+ProcessedMediaStream* GetDestination() { return mDest; }
+uint16_t InputNumber() const { return mInputNumber; }
+uint16_t OutputNumber() const { return mOutputNumber; }
+// Call on graph manager thread
+struct InputInterval {
+GraphTime mStart;
+GraphTime mEnd;
+bool mInputIsBlocked;
+};
+// Find the next time interval starting at or after aTime during which
+// mDest is not blocked and mSource's blocking status does not change.
+InputInterval GetNextInputInterval(GraphTime aTime);
+/**
+* Returns the graph that owns this port.
+*/
+MediaStreamGraphImpl* GraphImpl();
+MediaStreamGraph* Graph();
+/**
+* Sets the graph that owns this stream.  Should only be called once.
+*/
+void SetGraphImpl(MediaStreamGraphImpl* aGraph);
+size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const
+{
+size_t amount = 0;
+// Not owned:
+// - mSource
+// - mDest
+// - mGraph
+return amount;
+}
+size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const
+{
+return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
+}
+private:
+friend class MediaStreamGraphImpl;
+friend class MediaStream;
+friend class ProcessedMediaStream;
+// Never modified after Init()
+MediaStream* mSource;
+ProcessedMediaStream* mDest;
+uint32_t mFlags;
+// The input and output numbers are optional, and are currently only used by
+// Web Audio.
+const uint16_t mInputNumber;
+const uint16_t mOutputNumber;
+// Our media stream graph
+MediaStreamGraphImpl* mGraph;
+};
+/**
+* This stream processes zero or more input streams in parallel to produce
+* its output. The details of how the output is produced are handled by
+* subclasses overriding the ProcessInput method.
+*/
+class ProcessedMediaStream : public MediaStream {
+public:
+ProcessedMediaStream(DOMMediaStream* aWrapper)
+: MediaStream(aWrapper), mAutofinish(false), mInCycle(false)
+{}
+// Control API.
+/**
+* Allocates a new input port attached to source aStream.
+* This stream can be removed by calling MediaInputPort::Remove().
+*/
+already_AddRefed<MediaInputPort> AllocateInputPort(MediaStream* aStream,
+uint32_t aFlags = 0,
+uint16_t aInputNumber = 0,
+uint16_t aOutputNumber = 0);
+/**
+* Force this stream into the finished state.
+*/
+void Finish();
+/**
+* Set the autofinish flag on this stream (defaults to false). When this flag
+* is set, and all input streams are in the finished state (including if there
+* are no input streams), this stream automatically enters the finished state.
+*/
+void SetAutofinish(bool aAutofinish);
+virtual ProcessedMediaStream* AsProcessedStream() { return this; }
+friend class MediaStreamGraphImpl;
+// Do not call these from outside MediaStreamGraph.cpp!
+virtual void AddInput(MediaInputPort* aPort);
+virtual void RemoveInput(MediaInputPort* aPort)
+{
+mInputs.RemoveElement(aPort);
+}
+bool HasInputPort(MediaInputPort* aPort)
+{
+return mInputs.Contains(aPort);
+}
+uint32_t InputPortCount()
+{
+return mInputs.Length();
+}
+virtual void DestroyImpl();
+/**
+* This gets called after we've computed the blocking states for all
+* streams (mBlocked is up to date up to mStateComputedTime).
+* Also, we've produced output for all streams up to this one. If this stream
+* is not in a cycle, then all its source streams have produced data.
+* Generate output from aFrom to aTo.
+* This will be called on streams that have finished. Most stream types should
+* just return immediately if IsFinishedOnGraphThread(), but some may wish to
+* update internal state (see AudioNodeStream).
+* ProcessInput is allowed to call FinishOnGraphThread only if ALLOW_FINISH
+* is in aFlags. (This flag will be set when aTo >= mStateComputedTime, i.e.
+* when we've producing the last block of data we need to produce.) Otherwise
+* we can get into a situation where we've determined the stream should not
+* block before mStateComputedTime, but the stream finishes before
+* mStateComputedTime, violating the invariant that finished streams are blocked.
+*/
+enum {
+ALLOW_FINISH = 0x01
+};
+virtual void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) = 0;
+void SetAutofinishImpl(bool aAutofinish) { mAutofinish = aAutofinish; }
+/**
+* Forward SetTrackEnabled() to the input MediaStream(s) and translate the ID
+*/
+virtual void ForwardTrackEnabled(TrackID aOutputID, bool aEnabled) {};
+bool InCycle() const { return mInCycle; }
+virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const MOZ_OVERRIDE
+{
+size_t amount = MediaStream::SizeOfExcludingThis(aMallocSizeOf);
+// Not owned:
+// - mInputs elements
+amount += mInputs.SizeOfExcludingThis(aMallocSizeOf);
+return amount;
+}
+virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const MOZ_OVERRIDE
+{
+return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
+}
+protected:
+// This state is all accessed only on the media graph thread.
+// The list of all inputs that are currently enabled or waiting to be enabled.
+nsTArray<MediaInputPort*> mInputs;
+bool mAutofinish;
+// True if and only if this stream is in a cycle.
+// Updated by MediaStreamGraphImpl::UpdateStreamOrder.
+bool mInCycle;
+};
+/**
+* Initially, at least, we will have a singleton MediaStreamGraph per
+* process.  Each OfflineAudioContext object creates its own MediaStreamGraph
+* object too.
+*/
+class MediaStreamGraph {
+public:
+// We ensure that the graph current time advances in multiples of
+// IdealAudioBlockSize()/AudioStream::PreferredSampleRate(). A stream that
+// never blocks and has a track with the ideal audio rate will produce audio
+// in multiples of the block size.
+// Main thread only
+static MediaStreamGraph* GetInstance();
+static MediaStreamGraph* CreateNonRealtimeInstance(TrackRate aSampleRate);
+// Idempotent
+static void DestroyNonRealtimeInstance(MediaStreamGraph* aGraph);
+// Control API.
+/**
+* Create a stream that a media decoder (or some other source of
+* media data, such as a camera) can write to.
+*/
+SourceMediaStream* CreateSourceStream(DOMMediaStream* aWrapper);
+/**
+* Create a stream that will form the union of the tracks of its input
+* streams.
+* A TrackUnionStream contains all the tracks of all its input streams.
+* Adding a new input stream makes that stream's tracks immediately appear as new
+* tracks starting at the time the input stream was added.
+* Removing an input stream makes the output tracks corresponding to the
+* removed tracks immediately end.
+* For each added track, the track ID of the output track is the track ID
+* of the input track or one plus the maximum ID of all previously added
+* tracks, whichever is greater.
+* TODO at some point we will probably need to add API to select
+* particular tracks of each input stream.
+*/
+ProcessedMediaStream* CreateTrackUnionStream(DOMMediaStream* aWrapper);
+// Internal AudioNodeStreams can only pass their output to another
+// AudioNode, whereas external AudioNodeStreams can pass their output
+// to an nsAudioStream for playback.
+enum AudioNodeStreamKind { SOURCE_STREAM, INTERNAL_STREAM, EXTERNAL_STREAM };
+/**
+* Create a stream that will process audio for an AudioNode.
+* Takes ownership of aEngine.  aSampleRate is the sampling rate used
+* for the stream.  If 0 is passed, the sampling rate of the engine's
+* node will get used.
+*/
+AudioNodeStream* CreateAudioNodeStream(AudioNodeEngine* aEngine,
+AudioNodeStreamKind aKind,
+TrackRate aSampleRate = 0);
+AudioNodeExternalInputStream*
+CreateAudioNodeExternalInputStream(AudioNodeEngine* aEngine,
+TrackRate aSampleRate = 0);
+bool IsNonRealtime() const;
+/**
+* Start processing non-realtime for a specific number of ticks.
+*/
+void StartNonRealtimeProcessing(TrackRate aRate, uint32_t aTicksToProcess);
+/**
+* Media graph thread only.
+* Dispatches a runnable that will run on the main thread after all
+* main-thread stream state has been next updated.
+* Should only be called during MediaStreamListener callbacks or during
+* ProcessedMediaStream::ProcessInput().
+*/
+void DispatchToMainThreadAfterStreamStateUpdate(already_AddRefed<nsIRunnable> aRunnable)
+{
+*mPendingUpdateRunnables.AppendElement() = aRunnable;
+}
+protected:
+MediaStreamGraph()
+: mNextGraphUpdateIndex(1)
+{
+MOZ_COUNT_CTOR(MediaStreamGraph);
+}
+virtual ~MediaStreamGraph()
+{
+MOZ_COUNT_DTOR(MediaStreamGraph);
+}
+// Media graph thread only
+nsTArray<nsCOMPtr<nsIRunnable> > mPendingUpdateRunnables;
+// Main thread only
+// The number of updates we have sent to the media graph thread + 1.
+// We start this at 1 just to ensure that 0 is usable as a special value.
+int64_t mNextGraphUpdateIndex;
+};
+}
+#endif /* MOZILLA_MEDIASTREAMGRAPH_H_ */

The Tor Browser / file comparison

comparison: content/media/MediaStreamGraph.h

content/media/MediaStreamGraph.h