1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/content/media/MediaStreamGraph.h Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,1184 @@
1.4 +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
1.5 +/* This Source Code Form is subject to the terms of the Mozilla Public
1.6 + * License, v. 2.0. If a copy of the MPL was not distributed with this file,
1.7 + * You can obtain one at http://mozilla.org/MPL/2.0/. */
1.8 +
1.9 +#ifndef MOZILLA_MEDIASTREAMGRAPH_H_
1.10 +#define MOZILLA_MEDIASTREAMGRAPH_H_
1.11 +
1.12 +#include "mozilla/Mutex.h"
1.13 +#include "mozilla/LinkedList.h"
1.14 +#include "AudioStream.h"
1.15 +#include "nsTArray.h"
1.16 +#include "nsIRunnable.h"
1.17 +#include "StreamBuffer.h"
1.18 +#include "TimeVarying.h"
1.19 +#include "VideoFrameContainer.h"
1.20 +#include "VideoSegment.h"
1.21 +#include "MainThreadUtils.h"
1.22 +#include "nsAutoRef.h"
1.23 +#include "speex/speex_resampler.h"
1.24 +#include "AudioMixer.h"
1.25 +#include "mozilla/dom/AudioChannelBinding.h"
1.26 +
1.27 +class nsIRunnable;
1.28 +
1.29 +template <>
1.30 +class nsAutoRefTraits<SpeexResamplerState> : public nsPointerRefTraits<SpeexResamplerState>
1.31 +{
1.32 + public:
1.33 + static void Release(SpeexResamplerState* aState) { speex_resampler_destroy(aState); }
1.34 +};
1.35 +
1.36 +namespace mozilla {
1.37 +
1.38 +class DOMMediaStream;
1.39 +
1.40 +#ifdef PR_LOGGING
1.41 +extern PRLogModuleInfo* gMediaStreamGraphLog;
1.42 +#endif
1.43 +
1.44 +/**
1.45 + * Microseconds relative to the start of the graph timeline.
1.46 + */
1.47 +typedef int64_t GraphTime;
1.48 +const GraphTime GRAPH_TIME_MAX = MEDIA_TIME_MAX;
1.49 +
1.50 +/*
1.51 + * MediaStreamGraph is a framework for synchronized audio/video processing
1.52 + * and playback. It is designed to be used by other browser components such as
1.53 + * HTML media elements, media capture APIs, real-time media streaming APIs,
1.54 + * multitrack media APIs, and advanced audio APIs.
1.55 + *
1.56 + * The MediaStreamGraph uses a dedicated thread to process media --- the media
1.57 + * graph thread. This ensures that we can process media through the graph
1.58 + * without blocking on main-thread activity. The media graph is only modified
1.59 + * on the media graph thread, to ensure graph changes can be processed without
1.60 + * interfering with media processing. All interaction with the media graph
1.61 + * thread is done with message passing.
1.62 + *
1.63 + * APIs that modify the graph or its properties are described as "control APIs".
1.64 + * These APIs are asynchronous; they queue graph changes internally and
1.65 + * those changes are processed all-at-once by the MediaStreamGraph. The
1.66 + * MediaStreamGraph monitors the main thread event loop via nsIAppShell::RunInStableState
1.67 + * to ensure that graph changes from a single event loop task are always
1.68 + * processed all together. Control APIs should only be used on the main thread,
1.69 + * currently; we may be able to relax that later.
1.70 + *
1.71 + * To allow precise synchronization of times in the control API, the
1.72 + * MediaStreamGraph maintains a "media timeline". Control APIs that take or
1.73 + * return times use that timeline. Those times never advance during
1.74 + * an event loop task. This time is returned by MediaStreamGraph::GetCurrentTime().
1.75 + *
1.76 + * Media decoding, audio processing and media playback use thread-safe APIs to
1.77 + * the media graph to ensure they can continue while the main thread is blocked.
1.78 + *
1.79 + * When the graph is changed, we may need to throw out buffered data and
1.80 + * reprocess it. This is triggered automatically by the MediaStreamGraph.
1.81 + */
1.82 +
1.83 +class MediaStreamGraph;
1.84 +
1.85 +/**
1.86 + * This is a base class for media graph thread listener callbacks.
1.87 + * Override methods to be notified of audio or video data or changes in stream
1.88 + * state.
1.89 + *
1.90 + * This can be used by stream recorders or network connections that receive
1.91 + * stream input. It could also be used for debugging.
1.92 + *
1.93 + * All notification methods are called from the media graph thread. Overriders
1.94 + * of these methods are responsible for all synchronization. Beware!
1.95 + * These methods are called without the media graph monitor held, so
1.96 + * reentry into media graph methods is possible, although very much discouraged!
1.97 + * You should do something non-blocking and non-reentrant (e.g. dispatch an
1.98 + * event to some thread) and return.
1.99 + * The listener is not allowed to add/remove any listeners from the stream.
1.100 + *
1.101 + * When a listener is first attached, we guarantee to send a NotifyBlockingChanged
1.102 + * callback to notify of the initial blocking state. Also, if a listener is
1.103 + * attached to a stream that has already finished, we'll call NotifyFinished.
1.104 + */
1.105 +class MediaStreamListener {
1.106 +protected:
1.107 + // Protected destructor, to discourage deletion outside of Release():
1.108 + virtual ~MediaStreamListener() {}
1.109 +
1.110 +public:
1.111 + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStreamListener)
1.112 +
1.113 + enum Consumption {
1.114 + CONSUMED,
1.115 + NOT_CONSUMED
1.116 + };
1.117 + /**
1.118 + * Notify that the stream is hooked up and we'd like to start or stop receiving
1.119 + * data on it. Only fires on SourceMediaStreams.
1.120 + * The initial state is assumed to be NOT_CONSUMED.
1.121 + */
1.122 + virtual void NotifyConsumptionChanged(MediaStreamGraph* aGraph, Consumption aConsuming) {}
1.123 +
1.124 + /**
1.125 + * When a SourceMediaStream has pulling enabled, and the MediaStreamGraph
1.126 + * control loop is ready to pull, this gets called. A NotifyPull implementation
1.127 + * is allowed to call the SourceMediaStream methods that alter track
1.128 + * data. It is not allowed to make other MediaStream API calls, including
1.129 + * calls to add or remove MediaStreamListeners. It is not allowed to block
1.130 + * for any length of time.
1.131 + * aDesiredTime is the stream time we would like to get data up to. Data
1.132 + * beyond this point will not be played until NotifyPull runs again, so there's
1.133 + * not much point in providing it. Note that if the stream is blocked for
1.134 + * some reason, then data before aDesiredTime may not be played immediately.
1.135 + */
1.136 + virtual void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime) {}
1.137 +
1.138 + enum Blocking {
1.139 + BLOCKED,
1.140 + UNBLOCKED
1.141 + };
1.142 + /**
1.143 + * Notify that the blocking status of the stream changed. The initial state
1.144 + * is assumed to be BLOCKED.
1.145 + */
1.146 + virtual void NotifyBlockingChanged(MediaStreamGraph* aGraph, Blocking aBlocked) {}
1.147 +
1.148 + /**
1.149 + * Notify that the stream has data in each track
1.150 + * for the stream's current time. Once this state becomes true, it will
1.151 + * always be true since we block stream time from progressing to times where
1.152 + * there isn't data in each track.
1.153 + */
1.154 + virtual void NotifyHasCurrentData(MediaStreamGraph* aGraph) {}
1.155 +
1.156 + /**
1.157 + * Notify that the stream output is advancing. aCurrentTime is the graph's
1.158 + * current time. MediaStream::GraphTimeToStreamTime can be used to get the
1.159 + * stream time.
1.160 + */
1.161 + virtual void NotifyOutput(MediaStreamGraph* aGraph, GraphTime aCurrentTime) {}
1.162 +
1.163 + /**
1.164 + * Notify that the stream finished.
1.165 + */
1.166 + virtual void NotifyFinished(MediaStreamGraph* aGraph) {}
1.167 +
1.168 + /**
1.169 + * Notify that your listener has been removed, either due to RemoveListener(),
1.170 + * or due to the stream being destroyed. You will get no further notifications.
1.171 + */
1.172 + virtual void NotifyRemoved(MediaStreamGraph* aGraph) {}
1.173 +
1.174 + enum {
1.175 + TRACK_EVENT_CREATED = 0x01,
1.176 + TRACK_EVENT_ENDED = 0x02
1.177 + };
1.178 + /**
1.179 + * Notify that changes to one of the stream tracks have been queued.
1.180 + * aTrackEvents can be any combination of TRACK_EVENT_CREATED and
1.181 + * TRACK_EVENT_ENDED. aQueuedMedia is the data being added to the track
1.182 + * at aTrackOffset (relative to the start of the stream).
1.183 + */
1.184 + virtual void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
1.185 + TrackRate aTrackRate,
1.186 + TrackTicks aTrackOffset,
1.187 + uint32_t aTrackEvents,
1.188 + const MediaSegment& aQueuedMedia) {}
1.189 +};
1.190 +
1.191 +/**
1.192 + * This is a base class for media graph thread listener direct callbacks
1.193 + * from within AppendToTrack(). Note that your regular listener will
1.194 + * still get NotifyQueuedTrackChanges() callbacks from the MSG thread, so
1.195 + * you must be careful to ignore them if AddDirectListener was successful.
1.196 + */
1.197 +class MediaStreamDirectListener : public MediaStreamListener
1.198 +{
1.199 +public:
1.200 + virtual ~MediaStreamDirectListener() {}
1.201 +
1.202 + /*
1.203 + * This will be called on any MediaStreamDirectListener added to a
1.204 + * a SourceMediaStream when AppendToTrack() is called. The MediaSegment
1.205 + * will be the RawSegment (unresampled) if available in AppendToTrack().
1.206 + * Note that NotifyQueuedTrackChanges() calls will also still occur.
1.207 + */
1.208 + virtual void NotifyRealtimeData(MediaStreamGraph* aGraph, TrackID aID,
1.209 + TrackRate aTrackRate,
1.210 + TrackTicks aTrackOffset,
1.211 + uint32_t aTrackEvents,
1.212 + const MediaSegment& aMedia) {}
1.213 +};
1.214 +
1.215 +/**
1.216 + * This is a base class for main-thread listener callbacks.
1.217 + * This callback is invoked on the main thread when the main-thread-visible
1.218 + * state of a stream has changed.
1.219 + *
1.220 + * These methods are called with the media graph monitor held, so
1.221 + * reentry into general media graph methods is not possible.
1.222 + * You should do something non-blocking and non-reentrant (e.g. dispatch an
1.223 + * event) and return. DispatchFromMainThreadAfterNextStreamStateUpdate
1.224 + * would be a good choice.
1.225 + * The listener is allowed to synchronously remove itself from the stream, but
1.226 + * not add or remove any other listeners.
1.227 + */
1.228 +class MainThreadMediaStreamListener {
1.229 +public:
1.230 + virtual void NotifyMainThreadStateChanged() = 0;
1.231 +};
1.232 +
1.233 +/**
1.234 + * Helper struct used to keep track of memory usage by AudioNodes.
1.235 + */
1.236 +struct AudioNodeSizes
1.237 +{
1.238 + size_t mDomNode;
1.239 + size_t mStream;
1.240 + size_t mEngine;
1.241 + nsCString mNodeType;
1.242 +};
1.243 +
1.244 +class MediaStreamGraphImpl;
1.245 +class SourceMediaStream;
1.246 +class ProcessedMediaStream;
1.247 +class MediaInputPort;
1.248 +class AudioNodeEngine;
1.249 +class AudioNodeExternalInputStream;
1.250 +class AudioNodeStream;
1.251 +struct AudioChunk;
1.252 +
1.253 +/**
1.254 + * A stream of synchronized audio and video data. All (not blocked) streams
1.255 + * progress at the same rate --- "real time". Streams cannot seek. The only
1.256 + * operation readers can perform on a stream is to read the next data.
1.257 + *
1.258 + * Consumers of a stream can be reading from it at different offsets, but that
1.259 + * should only happen due to the order in which consumers are being run.
1.260 + * Those offsets must not diverge in the long term, otherwise we would require
1.261 + * unbounded buffering.
1.262 + *
1.263 + * Streams can be in a "blocked" state. While blocked, a stream does not
1.264 + * produce data. A stream can be explicitly blocked via the control API,
1.265 + * or implicitly blocked by whatever's generating it (e.g. an underrun in the
1.266 + * source resource), or implicitly blocked because something consuming it
1.267 + * blocks, or implicitly because it has finished.
1.268 + *
1.269 + * A stream can be in a "finished" state. "Finished" streams are permanently
1.270 + * blocked.
1.271 + *
1.272 + * Transitions into and out of the "blocked" and "finished" states are managed
1.273 + * by the MediaStreamGraph on the media graph thread.
1.274 + *
1.275 + * We buffer media data ahead of the consumers' reading offsets. It is possible
1.276 + * to have buffered data but still be blocked.
1.277 + *
1.278 + * Any stream can have its audio and video playing when requested. The media
1.279 + * stream graph plays audio by constructing audio output streams as necessary.
1.280 + * Video is played by setting video frames into an VideoFrameContainer at the right
1.281 + * time. To ensure video plays in sync with audio, make sure that the same
1.282 + * stream is playing both the audio and video.
1.283 + *
1.284 + * The data in a stream is managed by StreamBuffer. It consists of a set of
1.285 + * tracks of various types that can start and end over time.
1.286 + *
1.287 + * Streams are explicitly managed. The client creates them via
1.288 + * MediaStreamGraph::CreateInput/ProcessedMediaStream, and releases them by calling
1.289 + * Destroy() when no longer needed (actual destruction will be deferred).
1.290 + * The actual object is owned by the MediaStreamGraph. The basic idea is that
1.291 + * main thread objects will keep Streams alive as long as necessary (using the
1.292 + * cycle collector to clean up whenever needed).
1.293 + *
1.294 + * We make them refcounted only so that stream-related messages with MediaStream*
1.295 + * pointers can be sent to the main thread safely.
1.296 + *
1.297 + * The lifetimes of MediaStreams are controlled from the main thread.
1.298 + * For MediaStreams exposed to the DOM, the lifetime is controlled by the DOM
1.299 + * wrapper; the DOM wrappers own their associated MediaStreams. When a DOM
1.300 + * wrapper is destroyed, it sends a Destroy message for the associated
1.301 + * MediaStream and clears its reference (the last main-thread reference to
1.302 + * the object). When the Destroy message is processed on the graph
1.303 + * manager thread we immediately release the affected objects (disentangling them
1.304 + * from other objects as necessary).
1.305 + *
1.306 + * This could cause problems for media processing if a MediaStream is
1.307 + * destroyed while a downstream MediaStream is still using it. Therefore
1.308 + * the DOM wrappers must keep upstream MediaStreams alive as long as they
1.309 + * could be being used in the media graph.
1.310 + *
1.311 + * At any time, however, a set of MediaStream wrappers could be
1.312 + * collected via cycle collection. Destroy messages will be sent
1.313 + * for those objects in arbitrary order and the MediaStreamGraph has to be able
1.314 + * to handle this.
1.315 + */
1.316 +class MediaStream : public mozilla::LinkedListElement<MediaStream> {
1.317 +public:
1.318 + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStream)
1.319 +
1.320 + MediaStream(DOMMediaStream* aWrapper);
1.321 +
1.322 +protected:
1.323 + // Protected destructor, to discourage deletion outside of Release():
1.324 + virtual ~MediaStream()
1.325 + {
1.326 + MOZ_COUNT_DTOR(MediaStream);
1.327 + NS_ASSERTION(mMainThreadDestroyed, "Should have been destroyed already");
1.328 + NS_ASSERTION(mMainThreadListeners.IsEmpty(),
1.329 + "All main thread listeners should have been removed");
1.330 + }
1.331 +
1.332 +public:
1.333 + /**
1.334 + * Returns the graph that owns this stream.
1.335 + */
1.336 + MediaStreamGraphImpl* GraphImpl();
1.337 + MediaStreamGraph* Graph();
1.338 + /**
1.339 + * Sets the graph that owns this stream. Should only be called once.
1.340 + */
1.341 + void SetGraphImpl(MediaStreamGraphImpl* aGraph);
1.342 + void SetGraphImpl(MediaStreamGraph* aGraph);
1.343 +
1.344 + // Control API.
1.345 + // Since a stream can be played multiple ways, we need to combine independent
1.346 + // volume settings. The aKey parameter is used to keep volume settings
1.347 + // separate. Since the stream is always playing the same contents, only
1.348 + // a single audio output stream is used; the volumes are combined.
1.349 + // Currently only the first enabled audio track is played.
1.350 + // XXX change this so all enabled audio tracks are mixed and played.
1.351 + virtual void AddAudioOutput(void* aKey);
1.352 + virtual void SetAudioOutputVolume(void* aKey, float aVolume);
1.353 + virtual void RemoveAudioOutput(void* aKey);
1.354 + // Since a stream can be played multiple ways, we need to be able to
1.355 + // play to multiple VideoFrameContainers.
1.356 + // Only the first enabled video track is played.
1.357 + virtual void AddVideoOutput(VideoFrameContainer* aContainer);
1.358 + virtual void RemoveVideoOutput(VideoFrameContainer* aContainer);
1.359 + // Explicitly block. Useful for example if a media element is pausing
1.360 + // and we need to stop its stream emitting its buffered data.
1.361 + virtual void ChangeExplicitBlockerCount(int32_t aDelta);
1.362 + // Events will be dispatched by calling methods of aListener.
1.363 + virtual void AddListener(MediaStreamListener* aListener);
1.364 + virtual void RemoveListener(MediaStreamListener* aListener);
1.365 + // A disabled track has video replaced by black, and audio replaced by
1.366 + // silence.
1.367 + void SetTrackEnabled(TrackID aTrackID, bool aEnabled);
1.368 + // Events will be dispatched by calling methods of aListener. It is the
1.369 + // responsibility of the caller to remove aListener before it is destroyed.
1.370 + void AddMainThreadListener(MainThreadMediaStreamListener* aListener)
1.371 + {
1.372 + NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
1.373 + mMainThreadListeners.AppendElement(aListener);
1.374 + }
1.375 + // It's safe to call this even if aListener is not currently a listener;
1.376 + // the call will be ignored.
1.377 + void RemoveMainThreadListener(MainThreadMediaStreamListener* aListener)
1.378 + {
1.379 + NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
1.380 + mMainThreadListeners.RemoveElement(aListener);
1.381 + }
1.382 + /**
1.383 + * Ensure a runnable will run on the main thread after running all pending
1.384 + * updates that were sent from the graph thread or will be sent before the
1.385 + * graph thread receives the next graph update.
1.386 + *
1.387 + * If the graph has been shut down or destroyed, then the runnable will be
1.388 + * dispatched to the event queue immediately. If the graph is non-realtime
1.389 + * and has not started, then the runnable will be run
1.390 + * synchronously/immediately. (There are no pending updates in these
1.391 + * situations.)
1.392 + *
1.393 + * Main thread only.
1.394 + */
1.395 + void RunAfterPendingUpdates(nsRefPtr<nsIRunnable> aRunnable);
1.396 +
1.397 + // Signal that the client is done with this MediaStream. It will be deleted later.
1.398 + virtual void Destroy();
1.399 + // Returns the main-thread's view of how much data has been processed by
1.400 + // this stream.
1.401 + StreamTime GetCurrentTime()
1.402 + {
1.403 + NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
1.404 + return mMainThreadCurrentTime;
1.405 + }
1.406 + // Return the main thread's view of whether this stream has finished.
1.407 + bool IsFinished()
1.408 + {
1.409 + NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
1.410 + return mMainThreadFinished;
1.411 + }
1.412 + bool IsDestroyed()
1.413 + {
1.414 + NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
1.415 + return mMainThreadDestroyed;
1.416 + }
1.417 +
1.418 + friend class MediaStreamGraphImpl;
1.419 + friend class MediaInputPort;
1.420 + friend class AudioNodeExternalInputStream;
1.421 +
1.422 + virtual SourceMediaStream* AsSourceStream() { return nullptr; }
1.423 + virtual ProcessedMediaStream* AsProcessedStream() { return nullptr; }
1.424 + virtual AudioNodeStream* AsAudioNodeStream() { return nullptr; }
1.425 +
1.426 + // media graph thread only
1.427 + void Init();
1.428 + // These Impl methods perform the core functionality of the control methods
1.429 + // above, on the media graph thread.
1.430 + /**
1.431 + * Stop all stream activity and disconnect it from all inputs and outputs.
1.432 + * This must be idempotent.
1.433 + */
1.434 + virtual void DestroyImpl();
1.435 + StreamTime GetBufferEnd() { return mBuffer.GetEnd(); }
1.436 +#ifdef DEBUG
1.437 + void DumpTrackInfo() { return mBuffer.DumpTrackInfo(); }
1.438 +#endif
1.439 + void SetAudioOutputVolumeImpl(void* aKey, float aVolume);
1.440 + void AddAudioOutputImpl(void* aKey)
1.441 + {
1.442 + mAudioOutputs.AppendElement(AudioOutput(aKey));
1.443 + }
1.444 + void RemoveAudioOutputImpl(void* aKey);
1.445 + void AddVideoOutputImpl(already_AddRefed<VideoFrameContainer> aContainer)
1.446 + {
1.447 + *mVideoOutputs.AppendElement() = aContainer;
1.448 + }
1.449 + void RemoveVideoOutputImpl(VideoFrameContainer* aContainer)
1.450 + {
1.451 + mVideoOutputs.RemoveElement(aContainer);
1.452 + }
1.453 + void ChangeExplicitBlockerCountImpl(GraphTime aTime, int32_t aDelta)
1.454 + {
1.455 + mExplicitBlockerCount.SetAtAndAfter(aTime, mExplicitBlockerCount.GetAt(aTime) + aDelta);
1.456 + }
1.457 + void AddListenerImpl(already_AddRefed<MediaStreamListener> aListener);
1.458 + void RemoveListenerImpl(MediaStreamListener* aListener);
1.459 + void RemoveAllListenersImpl();
1.460 + void SetTrackEnabledImpl(TrackID aTrackID, bool aEnabled);
1.461 + /**
1.462 + * Returns true when this stream requires the contents of its inputs even if
1.463 + * its own outputs are not being consumed. This is used to signal inputs to
1.464 + * this stream that they are being consumed; when they're not being consumed,
1.465 + * we make some optimizations.
1.466 + */
1.467 + virtual bool IsIntrinsicallyConsumed() const
1.468 + {
1.469 + return !mAudioOutputs.IsEmpty() || !mVideoOutputs.IsEmpty();
1.470 + }
1.471 +
1.472 + void AddConsumer(MediaInputPort* aPort)
1.473 + {
1.474 + mConsumers.AppendElement(aPort);
1.475 + }
1.476 + void RemoveConsumer(MediaInputPort* aPort)
1.477 + {
1.478 + mConsumers.RemoveElement(aPort);
1.479 + }
1.480 + uint32_t ConsumerCount()
1.481 + {
1.482 + return mConsumers.Length();
1.483 + }
1.484 + const StreamBuffer& GetStreamBuffer() { return mBuffer; }
1.485 + GraphTime GetStreamBufferStartTime() { return mBufferStartTime; }
1.486 + /**
1.487 + * Convert graph time to stream time. aTime must be <= mStateComputedTime
1.488 + * to ensure we know exactly how much time this stream will be blocked during
1.489 + * the interval.
1.490 + */
1.491 + StreamTime GraphTimeToStreamTime(GraphTime aTime);
1.492 + /**
1.493 + * Convert graph time to stream time. aTime can be > mStateComputedTime,
1.494 + * in which case we optimistically assume the stream will not be blocked
1.495 + * after mStateComputedTime.
1.496 + */
1.497 + StreamTime GraphTimeToStreamTimeOptimistic(GraphTime aTime);
1.498 + /**
1.499 + * Convert stream time to graph time. The result can be > mStateComputedTime,
1.500 + * in which case we did the conversion optimistically assuming the stream
1.501 + * will not be blocked after mStateComputedTime.
1.502 + */
1.503 + GraphTime StreamTimeToGraphTime(StreamTime aTime);
1.504 + bool IsFinishedOnGraphThread() { return mFinished; }
1.505 + void FinishOnGraphThread();
1.506 + /**
1.507 + * Identify which graph update index we are currently processing.
1.508 + */
1.509 + int64_t GetProcessingGraphUpdateIndex();
1.510 +
1.511 + bool HasCurrentData() { return mHasCurrentData; }
1.512 +
1.513 + StreamBuffer::Track* EnsureTrack(TrackID aTrack, TrackRate aSampleRate);
1.514 +
1.515 + void ApplyTrackDisabling(TrackID aTrackID, MediaSegment* aSegment, MediaSegment* aRawSegment = nullptr);
1.516 +
1.517 + DOMMediaStream* GetWrapper()
1.518 + {
1.519 + NS_ASSERTION(NS_IsMainThread(), "Only use DOMMediaStream on main thread");
1.520 + return mWrapper;
1.521 + }
1.522 +
1.523 + // Return true if the main thread needs to observe updates from this stream.
1.524 + virtual bool MainThreadNeedsUpdates() const
1.525 + {
1.526 + return true;
1.527 + }
1.528 +
1.529 + virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const;
1.530 + virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const;
1.531 +
1.532 + void SetAudioChannelType(dom::AudioChannel aType) { mAudioChannelType = aType; }
1.533 +
1.534 +protected:
1.535 + virtual void AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime, GraphTime aBlockedTime)
1.536 + {
1.537 + mBufferStartTime += aBlockedTime;
1.538 + mGraphUpdateIndices.InsertTimeAtStart(aBlockedTime);
1.539 + mGraphUpdateIndices.AdvanceCurrentTime(aCurrentTime);
1.540 + mExplicitBlockerCount.AdvanceCurrentTime(aCurrentTime);
1.541 +
1.542 + mBuffer.ForgetUpTo(aCurrentTime - mBufferStartTime);
1.543 + }
1.544 +
1.545 + // This state is all initialized on the main thread but
1.546 + // otherwise modified only on the media graph thread.
1.547 +
1.548 + // Buffered data. The start of the buffer corresponds to mBufferStartTime.
1.549 + // Conceptually the buffer contains everything this stream has ever played,
1.550 + // but we forget some prefix of the buffered data to bound the space usage.
1.551 + StreamBuffer mBuffer;
1.552 + // The time when the buffered data could be considered to have started playing.
1.553 + // This increases over time to account for time the stream was blocked before
1.554 + // mCurrentTime.
1.555 + GraphTime mBufferStartTime;
1.556 +
1.557 + // Client-set volume of this stream
1.558 + struct AudioOutput {
1.559 + AudioOutput(void* aKey) : mKey(aKey), mVolume(1.0f) {}
1.560 + void* mKey;
1.561 + float mVolume;
1.562 + };
1.563 + nsTArray<AudioOutput> mAudioOutputs;
1.564 + nsTArray<nsRefPtr<VideoFrameContainer> > mVideoOutputs;
1.565 + // We record the last played video frame to avoid redundant setting
1.566 + // of the current video frame.
1.567 + VideoFrame mLastPlayedVideoFrame;
1.568 + // The number of times this stream has been explicitly blocked by the control
1.569 + // API, minus the number of times it has been explicitly unblocked.
1.570 + TimeVarying<GraphTime,uint32_t,0> mExplicitBlockerCount;
1.571 + nsTArray<nsRefPtr<MediaStreamListener> > mListeners;
1.572 + nsTArray<MainThreadMediaStreamListener*> mMainThreadListeners;
1.573 + nsTArray<TrackID> mDisabledTrackIDs;
1.574 +
1.575 + // Precomputed blocking status (over GraphTime).
1.576 + // This is only valid between the graph's mCurrentTime and
1.577 + // mStateComputedTime. The stream is considered to have
1.578 + // not been blocked before mCurrentTime (its mBufferStartTime is increased
1.579 + // as necessary to account for that time instead) --- this avoids us having to
1.580 + // record the entire history of the stream's blocking-ness in mBlocked.
1.581 + TimeVarying<GraphTime,bool,5> mBlocked;
1.582 + // Maps graph time to the graph update that affected this stream at that time
1.583 + TimeVarying<GraphTime,int64_t,0> mGraphUpdateIndices;
1.584 +
1.585 + // MediaInputPorts to which this is connected
1.586 + nsTArray<MediaInputPort*> mConsumers;
1.587 +
1.588 + // Where audio output is going. There is one AudioOutputStream per
1.589 + // audio track.
1.590 + struct AudioOutputStream {
1.591 + // When we started audio playback for this track.
1.592 + // Add mStream->GetPosition() to find the current audio playback position.
1.593 + GraphTime mAudioPlaybackStartTime;
1.594 + // Amount of time that we've wanted to play silence because of the stream
1.595 + // blocking.
1.596 + MediaTime mBlockedAudioTime;
1.597 + // Last tick written to the audio output.
1.598 + TrackTicks mLastTickWritten;
1.599 + RefPtr<AudioStream> mStream;
1.600 + TrackID mTrackID;
1.601 +
1.602 + size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const
1.603 + {
1.604 + size_t amount = 0;
1.605 + amount += mStream->SizeOfIncludingThis(aMallocSizeOf);
1.606 + return amount;
1.607 + }
1.608 + };
1.609 + nsTArray<AudioOutputStream> mAudioOutputStreams;
1.610 +
1.611 + /**
1.612 + * When true, this means the stream will be finished once all
1.613 + * buffered data has been consumed.
1.614 + */
1.615 + bool mFinished;
1.616 + /**
1.617 + * When true, mFinished is true and we've played all the data in this stream
1.618 + * and fired NotifyFinished notifications.
1.619 + */
1.620 + bool mNotifiedFinished;
1.621 + /**
1.622 + * When true, the last NotifyBlockingChanged delivered to the listeners
1.623 + * indicated that the stream is blocked.
1.624 + */
1.625 + bool mNotifiedBlocked;
1.626 + /**
1.627 + * True if some data can be present by this stream if/when it's unblocked.
1.628 + * Set by the stream itself on the MediaStreamGraph thread. Only changes
1.629 + * from false to true once a stream has data, since we won't
1.630 + * unblock it until there's more data.
1.631 + */
1.632 + bool mHasCurrentData;
1.633 + /**
1.634 + * True if mHasCurrentData is true and we've notified listeners.
1.635 + */
1.636 + bool mNotifiedHasCurrentData;
1.637 +
1.638 + // Temporary data for ordering streams by dependency graph
1.639 + bool mHasBeenOrdered;
1.640 + bool mIsOnOrderingStack;
1.641 + // True if the stream is being consumed (i.e. has track data being played,
1.642 + // or is feeding into some stream that is being consumed).
1.643 + bool mIsConsumed;
1.644 + // Temporary data for computing blocking status of streams
1.645 + // True if we've added this stream to the set of streams we're computing
1.646 + // blocking for.
1.647 + bool mInBlockingSet;
1.648 + // True if this stream should be blocked in this phase.
1.649 + bool mBlockInThisPhase;
1.650 +
1.651 + // This state is only used on the main thread.
1.652 + DOMMediaStream* mWrapper;
1.653 + // Main-thread views of state
1.654 + StreamTime mMainThreadCurrentTime;
1.655 + bool mMainThreadFinished;
1.656 + bool mMainThreadDestroyed;
1.657 +
1.658 + // Our media stream graph. null if destroyed on the graph thread.
1.659 + MediaStreamGraphImpl* mGraph;
1.660 +
1.661 + dom::AudioChannel mAudioChannelType;
1.662 +};
1.663 +
1.664 +/**
1.665 + * This is a stream into which a decoder can write audio and video.
1.666 + *
1.667 + * Audio and video can be written on any thread, but you probably want to
1.668 + * always write from the same thread to avoid unexpected interleavings.
1.669 + */
1.670 +class SourceMediaStream : public MediaStream {
1.671 +public:
1.672 + SourceMediaStream(DOMMediaStream* aWrapper) :
1.673 + MediaStream(aWrapper),
1.674 + mLastConsumptionState(MediaStreamListener::NOT_CONSUMED),
1.675 + mMutex("mozilla::media::SourceMediaStream"),
1.676 + mUpdateKnownTracksTime(0),
1.677 + mPullEnabled(false),
1.678 + mUpdateFinished(false)
1.679 + {}
1.680 +
1.681 + virtual SourceMediaStream* AsSourceStream() { return this; }
1.682 +
1.683 + // Media graph thread only
1.684 + virtual void DestroyImpl();
1.685 +
1.686 + // Call these on any thread.
1.687 + /**
1.688 + * Enable or disable pulling. When pulling is enabled, NotifyPull
1.689 + * gets called on MediaStreamListeners for this stream during the
1.690 + * MediaStreamGraph control loop. Pulling is initially disabled.
1.691 + * Due to unavoidable race conditions, after a call to SetPullEnabled(false)
1.692 + * it is still possible for a NotifyPull to occur.
1.693 + */
1.694 + void SetPullEnabled(bool aEnabled);
1.695 +
1.696 + void AddDirectListener(MediaStreamDirectListener* aListener);
1.697 + void RemoveDirectListener(MediaStreamDirectListener* aListener);
1.698 +
1.699 + /**
1.700 + * Add a new track to the stream starting at the given base time (which
1.701 + * must be greater than or equal to the last time passed to
1.702 + * AdvanceKnownTracksTime). Takes ownership of aSegment. aSegment should
1.703 + * contain data starting after aStart.
1.704 + */
1.705 + void AddTrack(TrackID aID, TrackRate aRate, TrackTicks aStart,
1.706 + MediaSegment* aSegment);
1.707 +
1.708 + /**
1.709 + * Append media data to a track. Ownership of aSegment remains with the caller,
1.710 + * but aSegment is emptied.
1.711 + * Returns false if the data was not appended because no such track exists
1.712 + * or the stream was already finished.
1.713 + */
1.714 + bool AppendToTrack(TrackID aID, MediaSegment* aSegment, MediaSegment *aRawSegment = nullptr);
1.715 + /**
1.716 + * Returns true if the buffer currently has enough data.
1.717 + * Returns false if there isn't enough data or if no such track exists.
1.718 + */
1.719 + bool HaveEnoughBuffered(TrackID aID);
1.720 + /**
1.721 + * Ensures that aSignalRunnable will be dispatched to aSignalThread
1.722 + * when we don't have enough buffered data in the track (which could be
1.723 + * immediately). Will dispatch the runnable immediately if the track
1.724 + * does not exist. No op if a runnable is already present for this track.
1.725 + */
1.726 + void DispatchWhenNotEnoughBuffered(TrackID aID,
1.727 + nsIEventTarget* aSignalThread, nsIRunnable* aSignalRunnable);
1.728 + /**
1.729 + * Indicate that a track has ended. Do not do any more API calls
1.730 + * affecting this track.
1.731 + * Ignored if the track does not exist.
1.732 + */
1.733 + void EndTrack(TrackID aID);
1.734 + /**
1.735 + * Indicate that no tracks will be added starting before time aKnownTime.
1.736 + * aKnownTime must be >= its value at the last call to AdvanceKnownTracksTime.
1.737 + */
1.738 + void AdvanceKnownTracksTime(StreamTime aKnownTime);
1.739 + /**
1.740 + * Indicate that this stream should enter the "finished" state. All tracks
1.741 + * must have been ended via EndTrack. The finish time of the stream is
1.742 + * when all tracks have ended.
1.743 + */
1.744 + void FinishWithLockHeld();
1.745 + void Finish()
1.746 + {
1.747 + MutexAutoLock lock(mMutex);
1.748 + FinishWithLockHeld();
1.749 + }
1.750 +
1.751 + // Overriding allows us to hold the mMutex lock while changing the track enable status
1.752 + void SetTrackEnabledImpl(TrackID aTrackID, bool aEnabled) {
1.753 + MutexAutoLock lock(mMutex);
1.754 + MediaStream::SetTrackEnabledImpl(aTrackID, aEnabled);
1.755 + }
1.756 +
1.757 + /**
1.758 + * End all tracks and Finish() this stream. Used to voluntarily revoke access
1.759 + * to a LocalMediaStream.
1.760 + */
1.761 + void EndAllTrackAndFinish();
1.762 +
1.763 + /**
1.764 + * Note: Only call from Media Graph thread (eg NotifyPull)
1.765 + *
1.766 + * Returns amount of time (data) that is currently buffered in the track,
1.767 + * assuming playout via PlayAudio or via a TrackUnion - note that
1.768 + * NotifyQueuedTrackChanges() on a SourceMediaStream will occur without
1.769 + * any "extra" buffering, but NotifyQueued TrackChanges() on a TrackUnion
1.770 + * will be buffered.
1.771 + */
1.772 + TrackTicks GetBufferedTicks(TrackID aID);
1.773 +
1.774 + void RegisterForAudioMixing();
1.775 +
1.776 + // XXX need a Reset API
1.777 +
1.778 + friend class MediaStreamGraphImpl;
1.779 +
1.780 +protected:
1.781 + struct ThreadAndRunnable {
1.782 + void Init(nsIEventTarget* aTarget, nsIRunnable* aRunnable)
1.783 + {
1.784 + mTarget = aTarget;
1.785 + mRunnable = aRunnable;
1.786 + }
1.787 +
1.788 + nsCOMPtr<nsIEventTarget> mTarget;
1.789 + nsCOMPtr<nsIRunnable> mRunnable;
1.790 + };
1.791 + enum TrackCommands {
1.792 + TRACK_CREATE = MediaStreamListener::TRACK_EVENT_CREATED,
1.793 + TRACK_END = MediaStreamListener::TRACK_EVENT_ENDED
1.794 + };
1.795 + /**
1.796 + * Data for each track that hasn't ended.
1.797 + */
1.798 + struct TrackData {
1.799 + TrackID mID;
1.800 + // Sample rate of the input data.
1.801 + TrackRate mInputRate;
1.802 + // Sample rate of the output data, always equal to the sample rate of the
1.803 + // graph.
1.804 + TrackRate mOutputRate;
1.805 + // Resampler if the rate of the input track does not match the
1.806 + // MediaStreamGraph's.
1.807 + nsAutoRef<SpeexResamplerState> mResampler;
1.808 + TrackTicks mStart;
1.809 + // Each time the track updates are flushed to the media graph thread,
1.810 + // this is cleared.
1.811 + uint32_t mCommands;
1.812 + // Each time the track updates are flushed to the media graph thread,
1.813 + // the segment buffer is emptied.
1.814 + nsAutoPtr<MediaSegment> mData;
1.815 + nsTArray<ThreadAndRunnable> mDispatchWhenNotEnough;
1.816 + bool mHaveEnough;
1.817 + };
1.818 +
1.819 + bool NeedsMixing();
1.820 +
1.821 + void ResampleAudioToGraphSampleRate(TrackData* aTrackData, MediaSegment* aSegment);
1.822 +
1.823 + TrackData* FindDataForTrack(TrackID aID)
1.824 + {
1.825 + for (uint32_t i = 0; i < mUpdateTracks.Length(); ++i) {
1.826 + if (mUpdateTracks[i].mID == aID) {
1.827 + return &mUpdateTracks[i];
1.828 + }
1.829 + }
1.830 + return nullptr;
1.831 + }
1.832 +
1.833 + /**
1.834 + * Notify direct consumers of new data to one of the stream tracks.
1.835 + * The data doesn't have to be resampled (though it may be). This is called
1.836 + * from AppendToTrack on the thread providing the data, and will call
1.837 + * the Listeners on this thread.
1.838 + */
1.839 + void NotifyDirectConsumers(TrackData *aTrack,
1.840 + MediaSegment *aSegment);
1.841 +
1.842 + // Media stream graph thread only
1.843 + MediaStreamListener::Consumption mLastConsumptionState;
1.844 +
1.845 + // This must be acquired *before* MediaStreamGraphImpl's lock, if they are
1.846 + // held together.
1.847 + Mutex mMutex;
1.848 + // protected by mMutex
1.849 + StreamTime mUpdateKnownTracksTime;
1.850 + nsTArray<TrackData> mUpdateTracks;
1.851 + nsTArray<nsRefPtr<MediaStreamDirectListener> > mDirectListeners;
1.852 + bool mPullEnabled;
1.853 + bool mUpdateFinished;
1.854 + bool mNeedsMixing;
1.855 +};
1.856 +
1.857 +/**
1.858 + * Represents a connection between a ProcessedMediaStream and one of its
1.859 + * input streams.
1.860 + * We make these refcounted so that stream-related messages with MediaInputPort*
1.861 + * pointers can be sent to the main thread safely.
1.862 + *
1.863 + * When a port's source or destination stream dies, the stream's DestroyImpl
1.864 + * calls MediaInputPort::Disconnect to disconnect the port from
1.865 + * the source and destination streams.
1.866 + *
1.867 + * The lifetimes of MediaInputPort are controlled from the main thread.
1.868 + * The media graph adds a reference to the port. When a MediaInputPort is no
1.869 + * longer needed, main-thread code sends a Destroy message for the port and
1.870 + * clears its reference (the last main-thread reference to the object). When
1.871 + * the Destroy message is processed on the graph manager thread we disconnect
1.872 + * the port and drop the graph's reference, destroying the object.
1.873 + */
1.874 +class MediaInputPort MOZ_FINAL {
1.875 +private:
1.876 + // Do not call this constructor directly. Instead call aDest->AllocateInputPort.
1.877 + MediaInputPort(MediaStream* aSource, ProcessedMediaStream* aDest,
1.878 + uint32_t aFlags, uint16_t aInputNumber,
1.879 + uint16_t aOutputNumber)
1.880 + : mSource(aSource)
1.881 + , mDest(aDest)
1.882 + , mFlags(aFlags)
1.883 + , mInputNumber(aInputNumber)
1.884 + , mOutputNumber(aOutputNumber)
1.885 + , mGraph(nullptr)
1.886 + {
1.887 + MOZ_COUNT_CTOR(MediaInputPort);
1.888 + }
1.889 +
1.890 + // Private destructor, to discourage deletion outside of Release():
1.891 + ~MediaInputPort()
1.892 + {
1.893 + MOZ_COUNT_DTOR(MediaInputPort);
1.894 + }
1.895 +
1.896 +public:
1.897 + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaInputPort)
1.898 +
1.899 + /**
1.900 + * The FLAG_BLOCK_INPUT and FLAG_BLOCK_OUTPUT flags can be used to control
1.901 + * exactly how the blocking statuses of the input and output streams affect
1.902 + * each other.
1.903 + */
1.904 + enum {
1.905 + // When set, blocking on the output stream forces blocking on the input
1.906 + // stream.
1.907 + FLAG_BLOCK_INPUT = 0x01,
1.908 + // When set, blocking on the input stream forces blocking on the output
1.909 + // stream.
1.910 + FLAG_BLOCK_OUTPUT = 0x02
1.911 + };
1.912 +
1.913 + // Called on graph manager thread
1.914 + // Do not call these from outside MediaStreamGraph.cpp!
1.915 + void Init();
1.916 + // Called during message processing to trigger removal of this stream.
1.917 + void Disconnect();
1.918 +
1.919 + // Control API
1.920 + /**
1.921 + * Disconnects and destroys the port. The caller must not reference this
1.922 + * object again.
1.923 + */
1.924 + void Destroy();
1.925 +
1.926 + // Any thread
1.927 + MediaStream* GetSource() { return mSource; }
1.928 + ProcessedMediaStream* GetDestination() { return mDest; }
1.929 +
1.930 + uint16_t InputNumber() const { return mInputNumber; }
1.931 + uint16_t OutputNumber() const { return mOutputNumber; }
1.932 +
1.933 + // Call on graph manager thread
1.934 + struct InputInterval {
1.935 + GraphTime mStart;
1.936 + GraphTime mEnd;
1.937 + bool mInputIsBlocked;
1.938 + };
1.939 + // Find the next time interval starting at or after aTime during which
1.940 + // mDest is not blocked and mSource's blocking status does not change.
1.941 + InputInterval GetNextInputInterval(GraphTime aTime);
1.942 +
1.943 + /**
1.944 + * Returns the graph that owns this port.
1.945 + */
1.946 + MediaStreamGraphImpl* GraphImpl();
1.947 + MediaStreamGraph* Graph();
1.948 + /**
1.949 + * Sets the graph that owns this stream. Should only be called once.
1.950 + */
1.951 + void SetGraphImpl(MediaStreamGraphImpl* aGraph);
1.952 +
1.953 + size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const
1.954 + {
1.955 + size_t amount = 0;
1.956 +
1.957 + // Not owned:
1.958 + // - mSource
1.959 + // - mDest
1.960 + // - mGraph
1.961 + return amount;
1.962 + }
1.963 +
1.964 + size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const
1.965 + {
1.966 + return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
1.967 + }
1.968 +
1.969 +private:
1.970 + friend class MediaStreamGraphImpl;
1.971 + friend class MediaStream;
1.972 + friend class ProcessedMediaStream;
1.973 + // Never modified after Init()
1.974 + MediaStream* mSource;
1.975 + ProcessedMediaStream* mDest;
1.976 + uint32_t mFlags;
1.977 + // The input and output numbers are optional, and are currently only used by
1.978 + // Web Audio.
1.979 + const uint16_t mInputNumber;
1.980 + const uint16_t mOutputNumber;
1.981 +
1.982 + // Our media stream graph
1.983 + MediaStreamGraphImpl* mGraph;
1.984 +};
1.985 +
1.986 +/**
1.987 + * This stream processes zero or more input streams in parallel to produce
1.988 + * its output. The details of how the output is produced are handled by
1.989 + * subclasses overriding the ProcessInput method.
1.990 + */
1.991 +class ProcessedMediaStream : public MediaStream {
1.992 +public:
1.993 + ProcessedMediaStream(DOMMediaStream* aWrapper)
1.994 + : MediaStream(aWrapper), mAutofinish(false), mInCycle(false)
1.995 + {}
1.996 +
1.997 + // Control API.
1.998 + /**
1.999 + * Allocates a new input port attached to source aStream.
1.1000 + * This stream can be removed by calling MediaInputPort::Remove().
1.1001 + */
1.1002 + already_AddRefed<MediaInputPort> AllocateInputPort(MediaStream* aStream,
1.1003 + uint32_t aFlags = 0,
1.1004 + uint16_t aInputNumber = 0,
1.1005 + uint16_t aOutputNumber = 0);
1.1006 + /**
1.1007 + * Force this stream into the finished state.
1.1008 + */
1.1009 + void Finish();
1.1010 + /**
1.1011 + * Set the autofinish flag on this stream (defaults to false). When this flag
1.1012 + * is set, and all input streams are in the finished state (including if there
1.1013 + * are no input streams), this stream automatically enters the finished state.
1.1014 + */
1.1015 + void SetAutofinish(bool aAutofinish);
1.1016 +
1.1017 + virtual ProcessedMediaStream* AsProcessedStream() { return this; }
1.1018 +
1.1019 + friend class MediaStreamGraphImpl;
1.1020 +
1.1021 + // Do not call these from outside MediaStreamGraph.cpp!
1.1022 + virtual void AddInput(MediaInputPort* aPort);
1.1023 + virtual void RemoveInput(MediaInputPort* aPort)
1.1024 + {
1.1025 + mInputs.RemoveElement(aPort);
1.1026 + }
1.1027 + bool HasInputPort(MediaInputPort* aPort)
1.1028 + {
1.1029 + return mInputs.Contains(aPort);
1.1030 + }
1.1031 + uint32_t InputPortCount()
1.1032 + {
1.1033 + return mInputs.Length();
1.1034 + }
1.1035 + virtual void DestroyImpl();
1.1036 + /**
1.1037 + * This gets called after we've computed the blocking states for all
1.1038 + * streams (mBlocked is up to date up to mStateComputedTime).
1.1039 + * Also, we've produced output for all streams up to this one. If this stream
1.1040 + * is not in a cycle, then all its source streams have produced data.
1.1041 + * Generate output from aFrom to aTo.
1.1042 + * This will be called on streams that have finished. Most stream types should
1.1043 + * just return immediately if IsFinishedOnGraphThread(), but some may wish to
1.1044 + * update internal state (see AudioNodeStream).
1.1045 + * ProcessInput is allowed to call FinishOnGraphThread only if ALLOW_FINISH
1.1046 + * is in aFlags. (This flag will be set when aTo >= mStateComputedTime, i.e.
1.1047 + * when we've producing the last block of data we need to produce.) Otherwise
1.1048 + * we can get into a situation where we've determined the stream should not
1.1049 + * block before mStateComputedTime, but the stream finishes before
1.1050 + * mStateComputedTime, violating the invariant that finished streams are blocked.
1.1051 + */
1.1052 + enum {
1.1053 + ALLOW_FINISH = 0x01
1.1054 + };
1.1055 + virtual void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) = 0;
1.1056 + void SetAutofinishImpl(bool aAutofinish) { mAutofinish = aAutofinish; }
1.1057 +
1.1058 + /**
1.1059 + * Forward SetTrackEnabled() to the input MediaStream(s) and translate the ID
1.1060 + */
1.1061 + virtual void ForwardTrackEnabled(TrackID aOutputID, bool aEnabled) {};
1.1062 +
1.1063 + bool InCycle() const { return mInCycle; }
1.1064 +
1.1065 + virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const MOZ_OVERRIDE
1.1066 + {
1.1067 + size_t amount = MediaStream::SizeOfExcludingThis(aMallocSizeOf);
1.1068 + // Not owned:
1.1069 + // - mInputs elements
1.1070 + amount += mInputs.SizeOfExcludingThis(aMallocSizeOf);
1.1071 + return amount;
1.1072 + }
1.1073 +
1.1074 + virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const MOZ_OVERRIDE
1.1075 + {
1.1076 + return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
1.1077 + }
1.1078 +
1.1079 +protected:
1.1080 + // This state is all accessed only on the media graph thread.
1.1081 +
1.1082 + // The list of all inputs that are currently enabled or waiting to be enabled.
1.1083 + nsTArray<MediaInputPort*> mInputs;
1.1084 + bool mAutofinish;
1.1085 + // True if and only if this stream is in a cycle.
1.1086 + // Updated by MediaStreamGraphImpl::UpdateStreamOrder.
1.1087 + bool mInCycle;
1.1088 +};
1.1089 +
1.1090 +/**
1.1091 + * Initially, at least, we will have a singleton MediaStreamGraph per
1.1092 + * process. Each OfflineAudioContext object creates its own MediaStreamGraph
1.1093 + * object too.
1.1094 + */
1.1095 +class MediaStreamGraph {
1.1096 +public:
1.1097 + // We ensure that the graph current time advances in multiples of
1.1098 + // IdealAudioBlockSize()/AudioStream::PreferredSampleRate(). A stream that
1.1099 + // never blocks and has a track with the ideal audio rate will produce audio
1.1100 + // in multiples of the block size.
1.1101 +
1.1102 + // Main thread only
1.1103 + static MediaStreamGraph* GetInstance();
1.1104 + static MediaStreamGraph* CreateNonRealtimeInstance(TrackRate aSampleRate);
1.1105 + // Idempotent
1.1106 + static void DestroyNonRealtimeInstance(MediaStreamGraph* aGraph);
1.1107 +
1.1108 + // Control API.
1.1109 + /**
1.1110 + * Create a stream that a media decoder (or some other source of
1.1111 + * media data, such as a camera) can write to.
1.1112 + */
1.1113 + SourceMediaStream* CreateSourceStream(DOMMediaStream* aWrapper);
1.1114 + /**
1.1115 + * Create a stream that will form the union of the tracks of its input
1.1116 + * streams.
1.1117 + * A TrackUnionStream contains all the tracks of all its input streams.
1.1118 + * Adding a new input stream makes that stream's tracks immediately appear as new
1.1119 + * tracks starting at the time the input stream was added.
1.1120 + * Removing an input stream makes the output tracks corresponding to the
1.1121 + * removed tracks immediately end.
1.1122 + * For each added track, the track ID of the output track is the track ID
1.1123 + * of the input track or one plus the maximum ID of all previously added
1.1124 + * tracks, whichever is greater.
1.1125 + * TODO at some point we will probably need to add API to select
1.1126 + * particular tracks of each input stream.
1.1127 + */
1.1128 + ProcessedMediaStream* CreateTrackUnionStream(DOMMediaStream* aWrapper);
1.1129 + // Internal AudioNodeStreams can only pass their output to another
1.1130 + // AudioNode, whereas external AudioNodeStreams can pass their output
1.1131 + // to an nsAudioStream for playback.
1.1132 + enum AudioNodeStreamKind { SOURCE_STREAM, INTERNAL_STREAM, EXTERNAL_STREAM };
1.1133 + /**
1.1134 + * Create a stream that will process audio for an AudioNode.
1.1135 + * Takes ownership of aEngine. aSampleRate is the sampling rate used
1.1136 + * for the stream. If 0 is passed, the sampling rate of the engine's
1.1137 + * node will get used.
1.1138 + */
1.1139 + AudioNodeStream* CreateAudioNodeStream(AudioNodeEngine* aEngine,
1.1140 + AudioNodeStreamKind aKind,
1.1141 + TrackRate aSampleRate = 0);
1.1142 +
1.1143 + AudioNodeExternalInputStream*
1.1144 + CreateAudioNodeExternalInputStream(AudioNodeEngine* aEngine,
1.1145 + TrackRate aSampleRate = 0);
1.1146 +
1.1147 + bool IsNonRealtime() const;
1.1148 + /**
1.1149 + * Start processing non-realtime for a specific number of ticks.
1.1150 + */
1.1151 + void StartNonRealtimeProcessing(TrackRate aRate, uint32_t aTicksToProcess);
1.1152 +
1.1153 + /**
1.1154 + * Media graph thread only.
1.1155 + * Dispatches a runnable that will run on the main thread after all
1.1156 + * main-thread stream state has been next updated.
1.1157 + * Should only be called during MediaStreamListener callbacks or during
1.1158 + * ProcessedMediaStream::ProcessInput().
1.1159 + */
1.1160 + void DispatchToMainThreadAfterStreamStateUpdate(already_AddRefed<nsIRunnable> aRunnable)
1.1161 + {
1.1162 + *mPendingUpdateRunnables.AppendElement() = aRunnable;
1.1163 + }
1.1164 +
1.1165 +protected:
1.1166 + MediaStreamGraph()
1.1167 + : mNextGraphUpdateIndex(1)
1.1168 + {
1.1169 + MOZ_COUNT_CTOR(MediaStreamGraph);
1.1170 + }
1.1171 + virtual ~MediaStreamGraph()
1.1172 + {
1.1173 + MOZ_COUNT_DTOR(MediaStreamGraph);
1.1174 + }
1.1175 +
1.1176 + // Media graph thread only
1.1177 + nsTArray<nsCOMPtr<nsIRunnable> > mPendingUpdateRunnables;
1.1178 +
1.1179 + // Main thread only
1.1180 + // The number of updates we have sent to the media graph thread + 1.
1.1181 + // We start this at 1 just to ensure that 0 is usable as a special value.
1.1182 + int64_t mNextGraphUpdateIndex;
1.1183 +};
1.1184 +
1.1185 +}
1.1186 +
1.1187 +#endif /* MOZILLA_MEDIASTREAMGRAPH_H_ */
