1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/content/media/MediaDecoderStateMachine.h Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,948 @@
1.4 +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
1.5 +/* vim:set ts=2 sw=2 sts=2 et cindent: */
1.6 +/* This Source Code Form is subject to the terms of the Mozilla Public
1.7 + * License, v. 2.0. If a copy of the MPL was not distributed with this
1.8 + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
1.9 +/*
1.10 +Each video element for a media file has two threads:
1.11 +
1.12 + 1) The Audio thread writes the decoded audio data to the audio
1.13 + hardware. This is done in a separate thread to ensure that the
1.14 + audio hardware gets a constant stream of data without
1.15 + interruption due to decoding or display. At some point
1.16 + AudioStream will be refactored to have a callback interface
1.17 + where it asks for data and an extra thread will no longer be
1.18 + needed.
1.19 +
1.20 + 2) The decode thread. This thread reads from the media stream and
1.21 + decodes the Theora and Vorbis data. It places the decoded data into
1.22 + queues for the other threads to pull from.
1.23 +
1.24 +All file reads, seeks, and all decoding must occur on the decode thread.
1.25 +Synchronisation of state between the thread is done via a monitor owned
1.26 +by MediaDecoder.
1.27 +
1.28 +The lifetime of the decode and audio threads is controlled by the state
1.29 +machine when it runs on the shared state machine thread. When playback
1.30 +needs to occur they are created and events dispatched to them to run
1.31 +them. These events exit when decoding/audio playback is completed or
1.32 +no longer required.
1.33 +
1.34 +A/V synchronisation is handled by the state machine. It examines the audio
1.35 +playback time and compares this to the next frame in the queue of video
1.36 +frames. If it is time to play the video frame it is then displayed, otherwise
1.37 +it schedules the state machine to run again at the time of the next frame.
1.38 +
1.39 +Frame skipping is done in the following ways:
1.40 +
1.41 + 1) The state machine will skip all frames in the video queue whose
1.42 + display time is less than the current audio time. This ensures
1.43 + the correct frame for the current time is always displayed.
1.44 +
1.45 + 2) The decode thread will stop decoding interframes and read to the
1.46 + next keyframe if it determines that decoding the remaining
1.47 + interframes will cause playback issues. It detects this by:
1.48 + a) If the amount of audio data in the audio queue drops
1.49 + below a threshold whereby audio may start to skip.
1.50 + b) If the video queue drops below a threshold where it
1.51 + will be decoding video data that won't be displayed due
1.52 + to the decode thread dropping the frame immediately.
1.53 +
1.54 +When hardware accelerated graphics is not available, YCbCr conversion
1.55 +is done on the decode thread when video frames are decoded.
1.56 +
1.57 +The decode thread pushes decoded audio and videos frames into two
1.58 +separate queues - one for audio and one for video. These are kept
1.59 +separate to make it easy to constantly feed audio data to the audio
1.60 +hardware while allowing frame skipping of video data. These queues are
1.61 +threadsafe, and neither the decode, audio, or state machine should
1.62 +be able to monopolize them, and cause starvation of the other threads.
1.63 +
1.64 +Both queues are bounded by a maximum size. When this size is reached
1.65 +the decode thread will no longer decode video or audio depending on the
1.66 +queue that has reached the threshold. If both queues are full, the decode
1.67 +thread will wait on the decoder monitor.
1.68 +
1.69 +When the decode queues are full (they've reaced their maximum size) and
1.70 +the decoder is not in PLAYING play state, the state machine may opt
1.71 +to shut down the decode thread in order to conserve resources.
1.72 +
1.73 +During playback the audio thread will be idle (via a Wait() on the
1.74 +monitor) if the audio queue is empty. Otherwise it constantly pops
1.75 +audio data off the queue and plays it with a blocking write to the audio
1.76 +hardware (via AudioStream).
1.77 +
1.78 +*/
1.79 +#if !defined(MediaDecoderStateMachine_h__)
1.80 +#define MediaDecoderStateMachine_h__
1.81 +
1.82 +#include "mozilla/Attributes.h"
1.83 +#include "nsThreadUtils.h"
1.84 +#include "MediaDecoder.h"
1.85 +#include "mozilla/ReentrantMonitor.h"
1.86 +#include "MediaDecoderReader.h"
1.87 +#include "MediaDecoderOwner.h"
1.88 +#include "MediaMetadataManager.h"
1.89 +
1.90 +class nsITimer;
1.91 +
1.92 +namespace mozilla {
1.93 +
1.94 +class AudioSegment;
1.95 +class VideoSegment;
1.96 +class MediaTaskQueue;
1.97 +class SharedThreadPool;
1.98 +
1.99 +// GetCurrentTime is defined in winbase.h as zero argument macro forwarding to
1.100 +// GetTickCount() and conflicts with MediaDecoderStateMachine::GetCurrentTime
1.101 +// implementation.
1.102 +#ifdef GetCurrentTime
1.103 +#undef GetCurrentTime
1.104 +#endif
1.105 +
1.106 +/*
1.107 + The state machine class. This manages the decoding and seeking in the
1.108 + MediaDecoderReader on the decode thread, and A/V sync on the shared
1.109 + state machine thread, and controls the audio "push" thread.
1.110 +
1.111 + All internal state is synchronised via the decoder monitor. State changes
1.112 + are either propagated by NotifyAll on the monitor (typically when state
1.113 + changes need to be propagated to non-state machine threads) or by scheduling
1.114 + the state machine to run another cycle on the shared state machine thread.
1.115 +
1.116 + See MediaDecoder.h for more details.
1.117 +*/
1.118 +class MediaDecoderStateMachine
1.119 +{
1.120 + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaDecoderStateMachine)
1.121 +public:
1.122 + typedef MediaDecoder::DecodedStreamData DecodedStreamData;
1.123 + MediaDecoderStateMachine(MediaDecoder* aDecoder,
1.124 + MediaDecoderReader* aReader,
1.125 + bool aRealTime = false);
1.126 +
1.127 + nsresult Init(MediaDecoderStateMachine* aCloneDonor);
1.128 +
1.129 + // Enumeration for the valid decoding states
1.130 + enum State {
1.131 + DECODER_STATE_DECODING_METADATA,
1.132 + DECODER_STATE_WAIT_FOR_RESOURCES,
1.133 + DECODER_STATE_DORMANT,
1.134 + DECODER_STATE_DECODING,
1.135 + DECODER_STATE_SEEKING,
1.136 + DECODER_STATE_BUFFERING,
1.137 + DECODER_STATE_COMPLETED,
1.138 + DECODER_STATE_SHUTDOWN
1.139 + };
1.140 +
1.141 + State GetState() {
1.142 + AssertCurrentThreadInMonitor();
1.143 + return mState;
1.144 + }
1.145 +
1.146 + // Set the audio volume. The decoder monitor must be obtained before
1.147 + // calling this.
1.148 + void SetVolume(double aVolume);
1.149 + void SetAudioCaptured(bool aCapture);
1.150 +
1.151 + // Check if the decoder needs to become dormant state.
1.152 + bool IsDormantNeeded();
1.153 + // Set/Unset dormant state.
1.154 + void SetDormant(bool aDormant);
1.155 + void Shutdown();
1.156 +
1.157 + // Called from the main thread to get the duration. The decoder monitor
1.158 + // must be obtained before calling this. It is in units of microseconds.
1.159 + int64_t GetDuration();
1.160 +
1.161 + // Called from the main thread to set the duration of the media resource
1.162 + // if it is able to be obtained via HTTP headers. Called from the
1.163 + // state machine thread to set the duration if it is obtained from the
1.164 + // media metadata. The decoder monitor must be obtained before calling this.
1.165 + // aDuration is in microseconds.
1.166 + void SetDuration(int64_t aDuration);
1.167 +
1.168 + // Called while decoding metadata to set the end time of the media
1.169 + // resource. The decoder monitor must be obtained before calling this.
1.170 + // aEndTime is in microseconds.
1.171 + void SetMediaEndTime(int64_t aEndTime);
1.172 +
1.173 + // Called from main thread to update the duration with an estimated value.
1.174 + // The duration is only changed if its significantly different than the
1.175 + // the current duration, as the incoming duration is an estimate and so
1.176 + // often is unstable as more data is read and the estimate is updated.
1.177 + // Can result in a durationchangeevent. aDuration is in microseconds.
1.178 + void UpdateEstimatedDuration(int64_t aDuration);
1.179 +
1.180 + // Functions used by assertions to ensure we're calling things
1.181 + // on the appropriate threads.
1.182 + bool OnDecodeThread() const;
1.183 + bool OnStateMachineThread() const;
1.184 + bool OnAudioThread() const {
1.185 + return IsCurrentThread(mAudioThread);
1.186 + }
1.187 +
1.188 + MediaDecoderOwner::NextFrameStatus GetNextFrameStatus();
1.189 +
1.190 + // Cause state transitions. These methods obtain the decoder monitor
1.191 + // to synchronise the change of state, and to notify other threads
1.192 + // that the state has changed.
1.193 + void Play();
1.194 +
1.195 + // Seeks to the decoder to aTarget asynchronously.
1.196 + void Seek(const SeekTarget& aTarget);
1.197 +
1.198 + // Returns the current playback position in seconds.
1.199 + // Called from the main thread to get the current frame time. The decoder
1.200 + // monitor must be obtained before calling this.
1.201 + double GetCurrentTime() const;
1.202 +
1.203 + // Clear the flag indicating that a playback position change event
1.204 + // is currently queued. This is called from the main thread and must
1.205 + // be called with the decode monitor held.
1.206 + void ClearPositionChangeFlag();
1.207 +
1.208 + // Called from the main thread or the decoder thread to set whether the media
1.209 + // resource can seek into unbuffered ranges. The decoder monitor must be
1.210 + // obtained before calling this.
1.211 + void SetTransportSeekable(bool aSeekable);
1.212 +
1.213 + // Called from the main thread or the decoder thread to set whether the media
1.214 + // can seek to random location. This is not true for chained ogg and WebM
1.215 + // media without index. The decoder monitor must be obtained before calling
1.216 + // this.
1.217 + void SetMediaSeekable(bool aSeekable);
1.218 +
1.219 + // Update the playback position. This can result in a timeupdate event
1.220 + // and an invalidate of the frame being dispatched asynchronously if
1.221 + // there is no such event currently queued.
1.222 + // Only called on the decoder thread. Must be called with
1.223 + // the decode monitor held.
1.224 + void UpdatePlaybackPosition(int64_t aTime);
1.225 +
1.226 + // Causes the state machine to switch to buffering state, and to
1.227 + // immediately stop playback and buffer downloaded data. Must be called
1.228 + // with the decode monitor held. Called on the state machine thread and
1.229 + // the main thread.
1.230 + void StartBuffering();
1.231 +
1.232 + // This is called on the state machine thread and audio thread.
1.233 + // The decoder monitor must be obtained before calling this.
1.234 + bool HasAudio() const {
1.235 + AssertCurrentThreadInMonitor();
1.236 + return mInfo.HasAudio();
1.237 + }
1.238 +
1.239 + // This is called on the state machine thread and audio thread.
1.240 + // The decoder monitor must be obtained before calling this.
1.241 + bool HasVideo() const {
1.242 + AssertCurrentThreadInMonitor();
1.243 + return mInfo.HasVideo();
1.244 + }
1.245 +
1.246 + // Should be called by main thread.
1.247 + bool HaveNextFrameData();
1.248 +
1.249 + // Must be called with the decode monitor held.
1.250 + bool IsBuffering() const {
1.251 + AssertCurrentThreadInMonitor();
1.252 +
1.253 + return mState == DECODER_STATE_BUFFERING;
1.254 + }
1.255 +
1.256 + // Must be called with the decode monitor held.
1.257 + bool IsSeeking() const {
1.258 + AssertCurrentThreadInMonitor();
1.259 +
1.260 + return mState == DECODER_STATE_SEEKING;
1.261 + }
1.262 +
1.263 + nsresult GetBuffered(dom::TimeRanges* aBuffered);
1.264 +
1.265 + void SetPlaybackRate(double aPlaybackRate);
1.266 + void SetPreservesPitch(bool aPreservesPitch);
1.267 +
1.268 + size_t SizeOfVideoQueue() {
1.269 + if (mReader) {
1.270 + return mReader->SizeOfVideoQueueInBytes();
1.271 + }
1.272 + return 0;
1.273 + }
1.274 +
1.275 + size_t SizeOfAudioQueue() {
1.276 + if (mReader) {
1.277 + return mReader->SizeOfAudioQueueInBytes();
1.278 + }
1.279 + return 0;
1.280 + }
1.281 +
1.282 + void NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64_t aOffset);
1.283 +
1.284 + int64_t GetEndMediaTime() const {
1.285 + AssertCurrentThreadInMonitor();
1.286 + return mEndTime;
1.287 + }
1.288 +
1.289 + bool IsTransportSeekable() {
1.290 + AssertCurrentThreadInMonitor();
1.291 + return mTransportSeekable;
1.292 + }
1.293 +
1.294 + bool IsMediaSeekable() {
1.295 + AssertCurrentThreadInMonitor();
1.296 + return mMediaSeekable;
1.297 + }
1.298 +
1.299 + // Returns the shared state machine thread.
1.300 + nsIEventTarget* GetStateMachineThread();
1.301 +
1.302 + // Calls ScheduleStateMachine() after taking the decoder lock. Also
1.303 + // notifies the decoder thread in case it's waiting on the decoder lock.
1.304 + void ScheduleStateMachineWithLockAndWakeDecoder();
1.305 +
1.306 + // Schedules the shared state machine thread to run the state machine
1.307 + // in aUsecs microseconds from now, if it's not already scheduled to run
1.308 + // earlier, in which case the request is discarded.
1.309 + nsresult ScheduleStateMachine(int64_t aUsecs = 0);
1.310 +
1.311 + // Timer function to implement ScheduleStateMachine(aUsecs).
1.312 + nsresult TimeoutExpired(int aGeneration);
1.313 +
1.314 + // Set the media fragment end time. aEndTime is in microseconds.
1.315 + void SetFragmentEndTime(int64_t aEndTime);
1.316 +
1.317 + // Drop reference to decoder. Only called during shutdown dance.
1.318 + void ReleaseDecoder() {
1.319 + MOZ_ASSERT(mReader);
1.320 + if (mReader) {
1.321 + mReader->ReleaseDecoder();
1.322 + }
1.323 + mDecoder = nullptr;
1.324 + }
1.325 +
1.326 + // If we're playing into a MediaStream, record the current point in the
1.327 + // MediaStream and the current point in our media resource so later we can
1.328 + // convert MediaStream playback positions to media resource positions. Best to
1.329 + // call this while we're not playing (while the MediaStream is blocked). Can
1.330 + // be called on any thread with the decoder monitor held.
1.331 + void SetSyncPointForMediaStream();
1.332 + int64_t GetCurrentTimeViaMediaStreamSync();
1.333 +
1.334 + // Copy queued audio/video data in the reader to any output MediaStreams that
1.335 + // need it.
1.336 + void SendStreamData();
1.337 + void FinishStreamData();
1.338 + bool HaveEnoughDecodedAudio(int64_t aAmpleAudioUSecs);
1.339 + bool HaveEnoughDecodedVideo();
1.340 +
1.341 + // Returns true if the state machine has shutdown or is in the process of
1.342 + // shutting down. The decoder monitor must be held while calling this.
1.343 + bool IsShutdown();
1.344 +
1.345 + void QueueMetadata(int64_t aPublishTime, int aChannels, int aRate, bool aHasAudio, bool aHasVideo, MetadataTags* aTags);
1.346 +
1.347 + // Returns true if we're currently playing. The decoder monitor must
1.348 + // be held.
1.349 + bool IsPlaying();
1.350 +
1.351 + // Called when the reader may have acquired the hardware resources required
1.352 + // to begin decoding. The state machine may move into DECODING_METADATA if
1.353 + // appropriate. The decoder monitor must be held while calling this.
1.354 + void NotifyWaitingForResourcesStatusChanged();
1.355 +
1.356 + // Notifies the state machine that should minimize the number of samples
1.357 + // decoded we preroll, until playback starts. The first time playback starts
1.358 + // the state machine is free to return to prerolling normally. Note
1.359 + // "prerolling" in this context refers to when we decode and buffer decoded
1.360 + // samples in advance of when they're needed for playback.
1.361 + void SetMinimizePrerollUntilPlaybackStarts();
1.362 +
1.363 +protected:
1.364 + virtual ~MediaDecoderStateMachine();
1.365 +
1.366 + void AssertCurrentThreadInMonitor() const { mDecoder->GetReentrantMonitor().AssertCurrentThreadIn(); }
1.367 +
1.368 + class WakeDecoderRunnable : public nsRunnable {
1.369 + public:
1.370 + WakeDecoderRunnable(MediaDecoderStateMachine* aSM)
1.371 + : mMutex("WakeDecoderRunnable"), mStateMachine(aSM) {}
1.372 + NS_IMETHOD Run() MOZ_OVERRIDE
1.373 + {
1.374 + nsRefPtr<MediaDecoderStateMachine> stateMachine;
1.375 + {
1.376 + // Don't let Run() (called by media stream graph thread) race with
1.377 + // Revoke() (called by decoder state machine thread)
1.378 + MutexAutoLock lock(mMutex);
1.379 + if (!mStateMachine)
1.380 + return NS_OK;
1.381 + stateMachine = mStateMachine;
1.382 + }
1.383 + stateMachine->ScheduleStateMachineWithLockAndWakeDecoder();
1.384 + return NS_OK;
1.385 + }
1.386 + void Revoke()
1.387 + {
1.388 + MutexAutoLock lock(mMutex);
1.389 + mStateMachine = nullptr;
1.390 + }
1.391 +
1.392 + Mutex mMutex;
1.393 + // Protected by mMutex.
1.394 + // We don't use an owning pointer here, because keeping mStateMachine alive
1.395 + // would mean in some cases we'd have to destroy mStateMachine from this
1.396 + // object, which would be problematic since MediaDecoderStateMachine can
1.397 + // only be destroyed on the main thread whereas this object can be destroyed
1.398 + // on the media stream graph thread.
1.399 + MediaDecoderStateMachine* mStateMachine;
1.400 + };
1.401 + WakeDecoderRunnable* GetWakeDecoderRunnable();
1.402 +
1.403 + MediaQueue<AudioData>& AudioQueue() { return mReader->AudioQueue(); }
1.404 + MediaQueue<VideoData>& VideoQueue() { return mReader->VideoQueue(); }
1.405 +
1.406 + // True if our buffers of decoded audio are not full, and we should
1.407 + // decode more.
1.408 + bool NeedToDecodeAudio();
1.409 +
1.410 + // Decodes some audio. This should be run on the decode task queue.
1.411 + void DecodeAudio();
1.412 +
1.413 + // True if our buffers of decoded video are not full, and we should
1.414 + // decode more.
1.415 + bool NeedToDecodeVideo();
1.416 +
1.417 + // Decodes some video. This should be run on the decode task queue.
1.418 + void DecodeVideo();
1.419 +
1.420 + // Returns true if we've got less than aAudioUsecs microseconds of decoded
1.421 + // and playable data. The decoder monitor must be held.
1.422 + bool HasLowDecodedData(int64_t aAudioUsecs);
1.423 +
1.424 + // Returns true if we're running low on data which is not yet decoded.
1.425 + // The decoder monitor must be held.
1.426 + bool HasLowUndecodedData();
1.427 +
1.428 + // Returns true if we have less than aUsecs of undecoded data available.
1.429 + bool HasLowUndecodedData(double aUsecs);
1.430 +
1.431 + // Returns the number of unplayed usecs of audio we've got decoded and/or
1.432 + // pushed to the hardware waiting to play. This is how much audio we can
1.433 + // play without having to run the audio decoder. The decoder monitor
1.434 + // must be held.
1.435 + int64_t AudioDecodedUsecs();
1.436 +
1.437 + // Returns true when there's decoded audio waiting to play.
1.438 + // The decoder monitor must be held.
1.439 + bool HasFutureAudio();
1.440 +
1.441 + // Returns true if we recently exited "quick buffering" mode.
1.442 + bool JustExitedQuickBuffering();
1.443 +
1.444 + // Waits on the decoder ReentrantMonitor for aUsecs microseconds. If the decoder
1.445 + // monitor is awoken by a Notify() call, we'll continue waiting, unless
1.446 + // we've moved into shutdown state. This enables us to ensure that we
1.447 + // wait for a specified time, and that the myriad of Notify()s we do on
1.448 + // the decoder monitor don't cause the audio thread to be starved. aUsecs
1.449 + // values of less than 1 millisecond are rounded up to 1 millisecond
1.450 + // (see bug 651023). The decoder monitor must be held. Called only on the
1.451 + // audio thread.
1.452 + void Wait(int64_t aUsecs);
1.453 +
1.454 + // Dispatches an asynchronous event to update the media element's ready state.
1.455 + void UpdateReadyState();
1.456 +
1.457 + // Resets playback timing data. Called when we seek, on the decode thread.
1.458 + void ResetPlayback();
1.459 +
1.460 + // Returns the audio clock, if we have audio, or -1 if we don't.
1.461 + // Called on the state machine thread.
1.462 + int64_t GetAudioClock();
1.463 +
1.464 + // Get the video stream position, taking the |playbackRate| change into
1.465 + // account. This is a position in the media, not the duration of the playback
1.466 + // so far.
1.467 + int64_t GetVideoStreamPosition();
1.468 +
1.469 + // Return the current time, either the audio clock if available (if the media
1.470 + // has audio, and the playback is possible), or a clock for the video.
1.471 + // Called on the state machine thread.
1.472 + int64_t GetClock();
1.473 +
1.474 + // Returns the presentation time of the first audio or video frame in the
1.475 + // media. If the media has video, it returns the first video frame. The
1.476 + // decoder monitor must be held with exactly one lock count. Called on the
1.477 + // state machine thread.
1.478 + VideoData* FindStartTime();
1.479 +
1.480 + // Update only the state machine's current playback position (and duration,
1.481 + // if unknown). Does not update the playback position on the decoder or
1.482 + // media element -- use UpdatePlaybackPosition for that. Called on the state
1.483 + // machine thread, caller must hold the decoder lock.
1.484 + void UpdatePlaybackPositionInternal(int64_t aTime);
1.485 +
1.486 + // Pushes the image down the rendering pipeline. Called on the shared state
1.487 + // machine thread. The decoder monitor must *not* be held when calling this.
1.488 + void RenderVideoFrame(VideoData* aData, TimeStamp aTarget);
1.489 +
1.490 + // If we have video, display a video frame if it's time for display has
1.491 + // arrived, otherwise sleep until it's time for the next frame. Update the
1.492 + // current frame time as appropriate, and trigger ready state update. The
1.493 + // decoder monitor must be held with exactly one lock count. Called on the
1.494 + // state machine thread.
1.495 + void AdvanceFrame();
1.496 +
1.497 + // Write aFrames of audio frames of silence to the audio hardware. Returns
1.498 + // the number of frames actually written. The write size is capped at
1.499 + // SILENCE_BYTES_CHUNK (32kB), so must be called in a loop to write the
1.500 + // desired number of frames. This ensures that the playback position
1.501 + // advances smoothly, and guarantees that we don't try to allocate an
1.502 + // impossibly large chunk of memory in order to play back silence. Called
1.503 + // on the audio thread.
1.504 + uint32_t PlaySilence(uint32_t aFrames,
1.505 + uint32_t aChannels,
1.506 + uint64_t aFrameOffset);
1.507 +
1.508 + // Pops an audio chunk from the front of the audio queue, and pushes its
1.509 + // audio data to the audio hardware.
1.510 + uint32_t PlayFromAudioQueue(uint64_t aFrameOffset, uint32_t aChannels);
1.511 +
1.512 + // Stops the audio thread. The decoder monitor must be held with exactly
1.513 + // one lock count. Called on the state machine thread.
1.514 + void StopAudioThread();
1.515 +
1.516 + // Starts the audio thread. The decoder monitor must be held with exactly
1.517 + // one lock count. Called on the state machine thread.
1.518 + nsresult StartAudioThread();
1.519 +
1.520 + // The main loop for the audio thread. Sent to the thread as
1.521 + // an nsRunnableMethod. This continually does blocking writes to
1.522 + // to audio stream to play audio data.
1.523 + void AudioLoop();
1.524 +
1.525 + // Sets internal state which causes playback of media to pause.
1.526 + // The decoder monitor must be held.
1.527 + void StopPlayback();
1.528 +
1.529 + // Sets internal state which causes playback of media to begin or resume.
1.530 + // Must be called with the decode monitor held.
1.531 + void StartPlayback();
1.532 +
1.533 + // Moves the decoder into decoding state. Called on the state machine
1.534 + // thread. The decoder monitor must be held.
1.535 + void StartDecoding();
1.536 +
1.537 + // Moves the decoder into the shutdown state, and dispatches an error
1.538 + // event to the media element. This begins shutting down the decoder.
1.539 + // The decoder monitor must be held. This is only called on the
1.540 + // decode thread.
1.541 + void DecodeError();
1.542 +
1.543 + void StartWaitForResources();
1.544 +
1.545 + // Dispatches a task to the decode task queue to begin decoding metadata.
1.546 + // This is threadsafe and can be called on any thread.
1.547 + // The decoder monitor must be held.
1.548 + nsresult EnqueueDecodeMetadataTask();
1.549 +
1.550 + nsresult DispatchAudioDecodeTaskIfNeeded();
1.551 +
1.552 + // Ensures a to decode audio has been dispatched to the decode task queue.
1.553 + // If a task to decode has already been dispatched, this does nothing,
1.554 + // otherwise this dispatches a task to do the decode.
1.555 + // This is called on the state machine or decode threads.
1.556 + // The decoder monitor must be held.
1.557 + nsresult EnsureAudioDecodeTaskQueued();
1.558 +
1.559 + nsresult DispatchVideoDecodeTaskIfNeeded();
1.560 +
1.561 + // Ensures a to decode video has been dispatched to the decode task queue.
1.562 + // If a task to decode has already been dispatched, this does nothing,
1.563 + // otherwise this dispatches a task to do the decode.
1.564 + // The decoder monitor must be held.
1.565 + nsresult EnsureVideoDecodeTaskQueued();
1.566 +
1.567 + // Dispatches a task to the decode task queue to seek the decoder.
1.568 + // The decoder monitor must be held.
1.569 + nsresult EnqueueDecodeSeekTask();
1.570 +
1.571 + // Calls the reader's SetIdle(), with aIsIdle as parameter. This is only
1.572 + // called in a task dispatched to the decode task queue, don't call it
1.573 + // directly.
1.574 + void SetReaderIdle();
1.575 + void SetReaderActive();
1.576 +
1.577 + // Re-evaluates the state and determines whether we need to dispatch
1.578 + // events to run the decode, or if not whether we should set the reader
1.579 + // to idle mode. This is threadsafe, and can be called from any thread.
1.580 + // The decoder monitor must be held.
1.581 + void DispatchDecodeTasksIfNeeded();
1.582 +
1.583 + // Called before we do anything on the decode task queue to set the reader
1.584 + // as not idle if it was idle. This is called before we decode, seek, or
1.585 + // decode metadata (in case we were dormant or awaiting resources).
1.586 + void EnsureActive();
1.587 +
1.588 + // Queries our state to see whether the decode has finished for all streams.
1.589 + // If so, we move into DECODER_STATE_COMPLETED and schedule the state machine
1.590 + // to run.
1.591 + // The decoder monitor must be held.
1.592 + void CheckIfDecodeComplete();
1.593 +
1.594 + // Returns the "media time". This is the absolute time which the media
1.595 + // playback has reached. i.e. this returns values in the range
1.596 + // [mStartTime, mEndTime], and mStartTime will not be 0 if the media does
1.597 + // not start at 0. Note this is different to the value returned
1.598 + // by GetCurrentTime(), which is in the range [0,duration].
1.599 + int64_t GetMediaTime() const {
1.600 + AssertCurrentThreadInMonitor();
1.601 + return mStartTime + mCurrentFrameTime;
1.602 + }
1.603 +
1.604 + // Returns an upper bound on the number of microseconds of audio that is
1.605 + // decoded and playable. This is the sum of the number of usecs of audio which
1.606 + // is decoded and in the reader's audio queue, and the usecs of unplayed audio
1.607 + // which has been pushed to the audio hardware for playback. Note that after
1.608 + // calling this, the audio hardware may play some of the audio pushed to
1.609 + // hardware, so this can only be used as a upper bound. The decoder monitor
1.610 + // must be held when calling this. Called on the decode thread.
1.611 + int64_t GetDecodedAudioDuration();
1.612 +
1.613 + // Load metadata. Called on the decode thread. The decoder monitor
1.614 + // must be held with exactly one lock count.
1.615 + nsresult DecodeMetadata();
1.616 +
1.617 + // Seeks to mSeekTarget. Called on the decode thread. The decoder monitor
1.618 + // must be held with exactly one lock count.
1.619 + void DecodeSeek();
1.620 +
1.621 + // Decode loop, decodes data until EOF or shutdown.
1.622 + // Called on the decode thread.
1.623 + void DecodeLoop();
1.624 +
1.625 + void CallDecodeMetadata();
1.626 +
1.627 + // Copy audio from an AudioData packet to aOutput. This may require
1.628 + // inserting silence depending on the timing of the audio packet.
1.629 + void SendStreamAudio(AudioData* aAudio, DecodedStreamData* aStream,
1.630 + AudioSegment* aOutput);
1.631 +
1.632 + // State machine thread run function. Defers to RunStateMachine().
1.633 + nsresult CallRunStateMachine();
1.634 +
1.635 + // Performs one "cycle" of the state machine. Polls the state, and may send
1.636 + // a video frame to be displayed, and generally manages the decode. Called
1.637 + // periodically via timer to ensure the video stays in sync.
1.638 + nsresult RunStateMachine();
1.639 +
1.640 + bool IsStateMachineScheduled() const {
1.641 + AssertCurrentThreadInMonitor();
1.642 + return !mTimeout.IsNull();
1.643 + }
1.644 +
1.645 + // Returns true if we're not playing and the decode thread has filled its
1.646 + // decode buffers and is waiting. We can shut the decode thread down in this
1.647 + // case as it may not be needed again.
1.648 + bool IsPausedAndDecoderWaiting();
1.649 +
1.650 + // The decoder object that created this state machine. The state machine
1.651 + // holds a strong reference to the decoder to ensure that the decoder stays
1.652 + // alive once media element has started the decoder shutdown process, and has
1.653 + // dropped its reference to the decoder. This enables the state machine to
1.654 + // keep using the decoder's monitor until the state machine has finished
1.655 + // shutting down, without fear of the monitor being destroyed. After
1.656 + // shutting down, the state machine will then release this reference,
1.657 + // causing the decoder to be destroyed. This is accessed on the decode,
1.658 + // state machine, audio and main threads.
1.659 + nsRefPtr<MediaDecoder> mDecoder;
1.660 +
1.661 + // The decoder monitor must be obtained before modifying this state.
1.662 + // NotifyAll on the monitor must be called when the state is changed so
1.663 + // that interested threads can wake up and alter behaviour if appropriate
1.664 + // Accessed on state machine, audio, main, and AV thread.
1.665 + State mState;
1.666 +
1.667 + // Thread for pushing audio onto the audio hardware.
1.668 + // The "audio push thread".
1.669 + nsCOMPtr<nsIThread> mAudioThread;
1.670 +
1.671 + // The task queue in which we run decode tasks. This is referred to as
1.672 + // the "decode thread", though in practise tasks can run on a different
1.673 + // thread every time they're called.
1.674 + RefPtr<MediaTaskQueue> mDecodeTaskQueue;
1.675 +
1.676 + RefPtr<SharedThreadPool> mStateMachineThreadPool;
1.677 +
1.678 + // Timer to run the state machine cycles. Used by
1.679 + // ScheduleStateMachine(). Access protected by decoder monitor.
1.680 + nsCOMPtr<nsITimer> mTimer;
1.681 +
1.682 + // Timestamp at which the next state machine cycle will run.
1.683 + // Access protected by decoder monitor.
1.684 + TimeStamp mTimeout;
1.685 +
1.686 + // Used to check if there are state machine cycles are running in sequence.
1.687 + DebugOnly<bool> mInRunningStateMachine;
1.688 +
1.689 + // The time that playback started from the system clock. This is used for
1.690 + // timing the presentation of video frames when there's no audio.
1.691 + // Accessed only via the state machine thread.
1.692 + TimeStamp mPlayStartTime;
1.693 +
1.694 + // When we start writing decoded data to a new DecodedDataStream, or we
1.695 + // restart writing due to PlaybackStarted(), we record where we are in the
1.696 + // MediaStream and what that corresponds to in the media.
1.697 + StreamTime mSyncPointInMediaStream;
1.698 + int64_t mSyncPointInDecodedStream; // microseconds
1.699 +
1.700 + // When the playbackRate changes, and there is no audio clock, it is necessary
1.701 + // to reset the mPlayStartTime. This is done next time the clock is queried,
1.702 + // when this member is true. Access protected by decoder monitor.
1.703 + bool mResetPlayStartTime;
1.704 +
1.705 + // The amount of time we've spent playing already the media. The current
1.706 + // playback position is therefore |Now() - mPlayStartTime +
1.707 + // mPlayDuration|, which must be adjusted by mStartTime if used with media
1.708 + // timestamps. Accessed only via the state machine thread.
1.709 + int64_t mPlayDuration;
1.710 +
1.711 + // Time that buffering started. Used for buffering timeout and only
1.712 + // accessed on the state machine thread. This is null while we're not
1.713 + // buffering.
1.714 + TimeStamp mBufferingStart;
1.715 +
1.716 + // Start time of the media, in microseconds. This is the presentation
1.717 + // time of the first frame decoded from the media, and is used to calculate
1.718 + // duration and as a bounds for seeking. Accessed on state machine, decode,
1.719 + // and main threads. Access controlled by decoder monitor.
1.720 + int64_t mStartTime;
1.721 +
1.722 + // Time of the last frame in the media, in microseconds. This is the
1.723 + // end time of the last frame in the media. Accessed on state
1.724 + // machine, decode, and main threads. Access controlled by decoder monitor.
1.725 + int64_t mEndTime;
1.726 +
1.727 + // Position to seek to in microseconds when the seek state transition occurs.
1.728 + // The decoder monitor lock must be obtained before reading or writing
1.729 + // this value. Accessed on main and decode thread.
1.730 + SeekTarget mSeekTarget;
1.731 +
1.732 + // Media Fragment end time in microseconds. Access controlled by decoder monitor.
1.733 + int64_t mFragmentEndTime;
1.734 +
1.735 + // The audio stream resource. Used on the state machine, and audio threads.
1.736 + // This is created and destroyed on the audio thread, while holding the
1.737 + // decoder monitor, so if this is used off the audio thread, you must
1.738 + // first acquire the decoder monitor and check that it is non-null.
1.739 + RefPtr<AudioStream> mAudioStream;
1.740 +
1.741 + // The reader, don't call its methods with the decoder monitor held.
1.742 + // This is created in the play state machine's constructor, and destroyed
1.743 + // in the play state machine's destructor.
1.744 + nsAutoPtr<MediaDecoderReader> mReader;
1.745 +
1.746 + // Accessed only on the state machine thread.
1.747 + // Not an nsRevocableEventPtr since we must Revoke() it well before
1.748 + // this object is destroyed, anyway.
1.749 + // Protected by decoder monitor except during the SHUTDOWN state after the
1.750 + // decoder thread has been stopped.
1.751 + nsRevocableEventPtr<WakeDecoderRunnable> mPendingWakeDecoder;
1.752 +
1.753 + // The time of the current frame in microseconds. This is referenced from
1.754 + // 0 which is the initial playback position. Set by the state machine
1.755 + // thread, and read-only from the main thread to get the current
1.756 + // time value. Synchronised via decoder monitor.
1.757 + int64_t mCurrentFrameTime;
1.758 +
1.759 + // The presentation time of the first audio frame that was played in
1.760 + // microseconds. We can add this to the audio stream position to determine
1.761 + // the current audio time. Accessed on audio and state machine thread.
1.762 + // Synchronized by decoder monitor.
1.763 + int64_t mAudioStartTime;
1.764 +
1.765 + // The end time of the last audio frame that's been pushed onto the audio
1.766 + // hardware in microseconds. This will approximately be the end time of the
1.767 + // audio stream, unless another frame is pushed to the hardware.
1.768 + int64_t mAudioEndTime;
1.769 +
1.770 + // The presentation end time of the last video frame which has been displayed
1.771 + // in microseconds. Accessed from the state machine thread.
1.772 + int64_t mVideoFrameEndTime;
1.773 +
1.774 + // Volume of playback. 0.0 = muted. 1.0 = full volume. Read/Written
1.775 + // from the state machine and main threads. Synchronised via decoder
1.776 + // monitor.
1.777 + double mVolume;
1.778 +
1.779 + // Playback rate. 1.0 : normal speed, 0.5 : two times slower. Synchronized via
1.780 + // decoder monitor.
1.781 + double mPlaybackRate;
1.782 +
1.783 + // Pitch preservation for the playback rate. Synchronized via decoder monitor.
1.784 + bool mPreservesPitch;
1.785 +
1.786 + // Position at which the last playback rate change occured, used to compute
1.787 + // the actual position in the stream when the playback rate changes and there
1.788 + // is no audio to be sync-ed to. Synchronized via decoder monitor.
1.789 + int64_t mBasePosition;
1.790 +
1.791 + // Time at which we started decoding. Synchronised via decoder monitor.
1.792 + TimeStamp mDecodeStartTime;
1.793 +
1.794 + // The maximum number of second we spend buffering when we are short on
1.795 + // unbuffered data.
1.796 + uint32_t mBufferingWait;
1.797 + int64_t mLowDataThresholdUsecs;
1.798 +
1.799 + // If we've got more than mAmpleVideoFrames decoded video frames waiting in
1.800 + // the video queue, we will not decode any more video frames until some have
1.801 + // been consumed by the play state machine thread.
1.802 + uint32_t mAmpleVideoFrames;
1.803 +
1.804 + // Low audio threshold. If we've decoded less than this much audio we
1.805 + // consider our audio decode "behind", and we may skip video decoding
1.806 + // in order to allow our audio decoding to catch up. We favour audio
1.807 + // decoding over video. We increase this threshold if we're slow to
1.808 + // decode video frames, in order to reduce the chance of audio underruns.
1.809 + // Note that we don't ever reset this threshold, it only ever grows as
1.810 + // we detect that the decode can't keep up with rendering.
1.811 + int64_t mLowAudioThresholdUsecs;
1.812 +
1.813 + // Our "ample" audio threshold. Once we've this much audio decoded, we
1.814 + // pause decoding. If we increase mLowAudioThresholdUsecs, we'll also
1.815 + // increase this too appropriately (we don't want mLowAudioThresholdUsecs
1.816 + // to be greater than ampleAudioThreshold, else we'd stop decoding!).
1.817 + // Note that we don't ever reset this threshold, it only ever grows as
1.818 + // we detect that the decode can't keep up with rendering.
1.819 + int64_t mAmpleAudioThresholdUsecs;
1.820 +
1.821 + // At the start of decoding we want to "preroll" the decode until we've
1.822 + // got a few frames decoded before we consider whether decode is falling
1.823 + // behind. Otherwise our "we're falling behind" logic will trigger
1.824 + // unneccessarily if we start playing as soon as the first sample is
1.825 + // decoded. These two fields store how many video frames and audio
1.826 + // samples we must consume before are considered to be finished prerolling.
1.827 + uint32_t mAudioPrerollUsecs;
1.828 + uint32_t mVideoPrerollFrames;
1.829 +
1.830 + // When we start decoding (either for the first time, or after a pause)
1.831 + // we may be low on decoded data. We don't want our "low data" logic to
1.832 + // kick in and decide that we're low on decoded data because the download
1.833 + // can't keep up with the decode, and cause us to pause playback. So we
1.834 + // have a "preroll" stage, where we ignore the results of our "low data"
1.835 + // logic during the first few frames of our decode. This occurs during
1.836 + // playback. The flags below are true when the corresponding stream is
1.837 + // being "prerolled".
1.838 + bool mIsAudioPrerolling;
1.839 + bool mIsVideoPrerolling;
1.840 +
1.841 + // True when we have an audio stream that we're decoding, and we have not
1.842 + // yet decoded to end of stream.
1.843 + bool mIsAudioDecoding;
1.844 +
1.845 + // True when we have a video stream that we're decoding, and we have not
1.846 + // yet decoded to end of stream.
1.847 + bool mIsVideoDecoding;
1.848 +
1.849 + // True when we have dispatched a task to the decode task queue to run
1.850 + // the audio decode.
1.851 + bool mDispatchedAudioDecodeTask;
1.852 +
1.853 + // True when we have dispatched a task to the decode task queue to run
1.854 + // the video decode.
1.855 + bool mDispatchedVideoDecodeTask;
1.856 +
1.857 + // True when the reader is initialized, but has been ordered "idle" by the
1.858 + // state machine. This happens when the MediaQueue's of decoded data are
1.859 + // "full" and playback is paused. The reader may choose to use the idle
1.860 + // notification to enter a low power state.
1.861 + bool mIsReaderIdle;
1.862 +
1.863 + // If the video decode is falling behind the audio, we'll start dropping the
1.864 + // inter-frames up until the next keyframe which is at or before the current
1.865 + // playback position. skipToNextKeyframe is true if we're currently
1.866 + // skipping up to the next keyframe.
1.867 + bool mSkipToNextKeyFrame;
1.868 +
1.869 + // True if we shouldn't play our audio (but still write it to any capturing
1.870 + // streams). When this is true, mStopAudioThread is always true and
1.871 + // the audio thread will never start again after it has stopped.
1.872 + bool mAudioCaptured;
1.873 +
1.874 + // True if the media resource can be seeked on a transport level. Accessed
1.875 + // from the state machine and main threads. Synchronised via decoder monitor.
1.876 + bool mTransportSeekable;
1.877 +
1.878 + // True if the media can be seeked. Accessed from the state machine and main
1.879 + // threads. Synchronised via decoder monitor.
1.880 + bool mMediaSeekable;
1.881 +
1.882 + // True if an event to notify about a change in the playback
1.883 + // position has been queued, but not yet run. It is set to false when
1.884 + // the event is run. This allows coalescing of these events as they can be
1.885 + // produced many times per second. Synchronised via decoder monitor.
1.886 + // Accessed on main and state machine threads.
1.887 + bool mPositionChangeQueued;
1.888 +
1.889 + // True if the audio playback thread has finished. It is finished
1.890 + // when either all the audio frames have completed playing, or we've moved
1.891 + // into shutdown state, and the threads are to be
1.892 + // destroyed. Written by the audio playback thread and read and written by
1.893 + // the state machine thread. Synchronised via decoder monitor.
1.894 + // When data is being sent to a MediaStream, this is true when all data has
1.895 + // been written to the MediaStream.
1.896 + bool mAudioCompleted;
1.897 +
1.898 + // True if mDuration has a value obtained from an HTTP header, or from
1.899 + // the media index/metadata. Accessed on the state machine thread.
1.900 + bool mGotDurationFromMetaData;
1.901 +
1.902 + // True if we've dispatched an event to the decode task queue to call
1.903 + // DecodeThreadRun(). We use this flag to prevent us from dispatching
1.904 + // unneccessary runnables, since the decode thread runs in a loop.
1.905 + bool mDispatchedEventToDecode;
1.906 +
1.907 + // False while audio thread should be running. Accessed state machine
1.908 + // and audio threads. Syncrhonised by decoder monitor.
1.909 + bool mStopAudioThread;
1.910 +
1.911 + // If this is true while we're in buffering mode, we can exit early,
1.912 + // as it's likely we may be able to playback. This happens when we enter
1.913 + // buffering mode soon after the decode starts, because the decode-ahead
1.914 + // ran fast enough to exhaust all data while the download is starting up.
1.915 + // Synchronised via decoder monitor.
1.916 + bool mQuickBuffering;
1.917 +
1.918 + // True if we should not decode/preroll unnecessary samples, unless we're
1.919 + // played. "Prerolling" in this context refers to when we decode and
1.920 + // buffer decoded samples in advance of when they're needed for playback.
1.921 + // This flag is set for preload=metadata media, and means we won't
1.922 + // decode more than the first video frame and first block of audio samples
1.923 + // for that media when we startup, or after a seek. When Play() is called,
1.924 + // we reset this flag, as we assume the user is playing the media, so
1.925 + // prerolling is appropriate then. This flag is used to reduce the overhead
1.926 + // of prerolling samples for media elements that may not play, both
1.927 + // memory and CPU overhead.
1.928 + bool mMinimizePreroll;
1.929 +
1.930 + // True if the decode thread has gone filled its buffers and is now
1.931 + // waiting to be awakened before it continues decoding. Synchronized
1.932 + // by the decoder monitor.
1.933 + bool mDecodeThreadWaiting;
1.934 +
1.935 + // True is we are decoding a realtime stream, like a camera stream
1.936 + bool mRealTime;
1.937 +
1.938 + // Stores presentation info required for playback. The decoder monitor
1.939 + // must be held when accessing this.
1.940 + MediaInfo mInfo;
1.941 +
1.942 + mozilla::MediaMetadataManager mMetadataManager;
1.943 +
1.944 + MediaDecoderOwner::NextFrameStatus mLastFrameStatus;
1.945 +
1.946 + // The id of timer tasks, used to ignore tasks that are scheduled previously.
1.947 + int mTimerId;
1.948 +};
1.949 +
1.950 +} // namespace mozilla;
1.951 +#endif
