michael@0: /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
michael@0: /* vim:set ts=2 sw=2 sts=2 et cindent: */
michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: /*
michael@0: Each video element for a media file has two threads:
michael@0: 
michael@0:   1) The Audio thread writes the decoded audio data to the audio
michael@0:      hardware. This is done in a separate thread to ensure that the
michael@0:      audio hardware gets a constant stream of data without
michael@0:      interruption due to decoding or display. At some point
michael@0:      AudioStream will be refactored to have a callback interface
michael@0:      where it asks for data and an extra thread will no longer be
michael@0:      needed.
michael@0: 
michael@0:   2) The decode thread. This thread reads from the media stream and
michael@0:      decodes the Theora and Vorbis data. It places the decoded data into
michael@0:      queues for the other threads to pull from.
michael@0: 
michael@0: All file reads, seeks, and all decoding must occur on the decode thread.
michael@0: Synchronisation of state between the thread is done via a monitor owned
michael@0: by MediaDecoder.
michael@0: 
michael@0: The lifetime of the decode and audio threads is controlled by the state
michael@0: machine when it runs on the shared state machine thread. When playback
michael@0: needs to occur they are created and events dispatched to them to run
michael@0: them. These events exit when decoding/audio playback is completed or
michael@0: no longer required.
michael@0: 
michael@0: A/V synchronisation is handled by the state machine. It examines the audio
michael@0: playback time and compares this to the next frame in the queue of video
michael@0: frames. If it is time to play the video frame it is then displayed, otherwise
michael@0: it schedules the state machine to run again at the time of the next frame.
michael@0: 
michael@0: Frame skipping is done in the following ways:
michael@0: 
michael@0:   1) The state machine will skip all frames in the video queue whose
michael@0:      display time is less than the current audio time. This ensures
michael@0:      the correct frame for the current time is always displayed.
michael@0: 
michael@0:   2) The decode thread will stop decoding interframes and read to the
michael@0:      next keyframe if it determines that decoding the remaining
michael@0:      interframes will cause playback issues. It detects this by:
michael@0:        a) If the amount of audio data in the audio queue drops
michael@0:           below a threshold whereby audio may start to skip.
michael@0:        b) If the video queue drops below a threshold where it
michael@0:           will be decoding video data that won't be displayed due
michael@0:           to the decode thread dropping the frame immediately.
michael@0: 
michael@0: When hardware accelerated graphics is not available, YCbCr conversion
michael@0: is done on the decode thread when video frames are decoded.
michael@0: 
michael@0: The decode thread pushes decoded audio and videos frames into two
michael@0: separate queues - one for audio and one for video. These are kept
michael@0: separate to make it easy to constantly feed audio data to the audio
michael@0: hardware while allowing frame skipping of video data. These queues are
michael@0: threadsafe, and neither the decode, audio, or state machine should
michael@0: be able to monopolize them, and cause starvation of the other threads.
michael@0: 
michael@0: Both queues are bounded by a maximum size. When this size is reached
michael@0: the decode thread will no longer decode video or audio depending on the
michael@0: queue that has reached the threshold. If both queues are full, the decode
michael@0: thread will wait on the decoder monitor.
michael@0: 
michael@0: When the decode queues are full (they've reaced their maximum size) and
michael@0: the decoder is not in PLAYING play state, the state machine may opt
michael@0: to shut down the decode thread in order to conserve resources.
michael@0: 
michael@0: During playback the audio thread will be idle (via a Wait() on the
michael@0: monitor) if the audio queue is empty. Otherwise it constantly pops
michael@0: audio data off the queue and plays it with a blocking write to the audio
michael@0: hardware (via AudioStream).
michael@0: 
michael@0: */
michael@0: #if !defined(MediaDecoderStateMachine_h__)
michael@0: #define MediaDecoderStateMachine_h__
michael@0: 
michael@0: #include "mozilla/Attributes.h"
michael@0: #include "nsThreadUtils.h"
michael@0: #include "MediaDecoder.h"
michael@0: #include "mozilla/ReentrantMonitor.h"
michael@0: #include "MediaDecoderReader.h"
michael@0: #include "MediaDecoderOwner.h"
michael@0: #include "MediaMetadataManager.h"
michael@0: 
michael@0: class nsITimer;
michael@0: 
michael@0: namespace mozilla {
michael@0: 
michael@0: class AudioSegment;
michael@0: class VideoSegment;
michael@0: class MediaTaskQueue;
michael@0: class SharedThreadPool;
michael@0: 
michael@0: // GetCurrentTime is defined in winbase.h as zero argument macro forwarding to
michael@0: // GetTickCount() and conflicts with MediaDecoderStateMachine::GetCurrentTime
michael@0: // implementation.
michael@0: #ifdef GetCurrentTime
michael@0: #undef GetCurrentTime
michael@0: #endif
michael@0: 
michael@0: /*
michael@0:   The state machine class. This manages the decoding and seeking in the
michael@0:   MediaDecoderReader on the decode thread, and A/V sync on the shared
michael@0:   state machine thread, and controls the audio "push" thread.
michael@0: 
michael@0:   All internal state is synchronised via the decoder monitor. State changes
michael@0:   are either propagated by NotifyAll on the monitor (typically when state
michael@0:   changes need to be propagated to non-state machine threads) or by scheduling
michael@0:   the state machine to run another cycle on the shared state machine thread.
michael@0: 
michael@0:   See MediaDecoder.h for more details.
michael@0: */
michael@0: class MediaDecoderStateMachine
michael@0: {
michael@0:   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaDecoderStateMachine)
michael@0: public:
michael@0:   typedef MediaDecoder::DecodedStreamData DecodedStreamData;
michael@0:   MediaDecoderStateMachine(MediaDecoder* aDecoder,
michael@0:                                MediaDecoderReader* aReader,
michael@0:                                bool aRealTime = false);
michael@0: 
michael@0:   nsresult Init(MediaDecoderStateMachine* aCloneDonor);
michael@0: 
michael@0:   // Enumeration for the valid decoding states
michael@0:   enum State {
michael@0:     DECODER_STATE_DECODING_METADATA,
michael@0:     DECODER_STATE_WAIT_FOR_RESOURCES,
michael@0:     DECODER_STATE_DORMANT,
michael@0:     DECODER_STATE_DECODING,
michael@0:     DECODER_STATE_SEEKING,
michael@0:     DECODER_STATE_BUFFERING,
michael@0:     DECODER_STATE_COMPLETED,
michael@0:     DECODER_STATE_SHUTDOWN
michael@0:   };
michael@0: 
michael@0:   State GetState() {
michael@0:     AssertCurrentThreadInMonitor();
michael@0:     return mState;
michael@0:   }
michael@0: 
michael@0:   // Set the audio volume. The decoder monitor must be obtained before
michael@0:   // calling this.
michael@0:   void SetVolume(double aVolume);
michael@0:   void SetAudioCaptured(bool aCapture);
michael@0: 
michael@0:   // Check if the decoder needs to become dormant state.
michael@0:   bool IsDormantNeeded();
michael@0:   // Set/Unset dormant state.
michael@0:   void SetDormant(bool aDormant);
michael@0:   void Shutdown();
michael@0: 
michael@0:   // Called from the main thread to get the duration. The decoder monitor
michael@0:   // must be obtained before calling this. It is in units of microseconds.
michael@0:   int64_t GetDuration();
michael@0: 
michael@0:   // Called from the main thread to set the duration of the media resource
michael@0:   // if it is able to be obtained via HTTP headers. Called from the
michael@0:   // state machine thread to set the duration if it is obtained from the
michael@0:   // media metadata. The decoder monitor must be obtained before calling this.
michael@0:   // aDuration is in microseconds.
michael@0:   void SetDuration(int64_t aDuration);
michael@0: 
michael@0:   // Called while decoding metadata to set the end time of the media
michael@0:   // resource. The decoder monitor must be obtained before calling this.
michael@0:   // aEndTime is in microseconds.
michael@0:   void SetMediaEndTime(int64_t aEndTime);
michael@0: 
michael@0:   // Called from main thread to update the duration with an estimated value.
michael@0:   // The duration is only changed if its significantly different than the
michael@0:   // the current duration, as the incoming duration is an estimate and so
michael@0:   // often is unstable as more data is read and the estimate is updated.
michael@0:   // Can result in a durationchangeevent. aDuration is in microseconds.
michael@0:   void UpdateEstimatedDuration(int64_t aDuration);
michael@0: 
michael@0:   // Functions used by assertions to ensure we're calling things
michael@0:   // on the appropriate threads.
michael@0:   bool OnDecodeThread() const;
michael@0:   bool OnStateMachineThread() const;
michael@0:   bool OnAudioThread() const {
michael@0:     return IsCurrentThread(mAudioThread);
michael@0:   }
michael@0: 
michael@0:   MediaDecoderOwner::NextFrameStatus GetNextFrameStatus();
michael@0: 
michael@0:   // Cause state transitions. These methods obtain the decoder monitor
michael@0:   // to synchronise the change of state, and to notify other threads
michael@0:   // that the state has changed.
michael@0:   void Play();
michael@0: 
michael@0:   // Seeks to the decoder to aTarget asynchronously.
michael@0:   void Seek(const SeekTarget& aTarget);
michael@0: 
michael@0:   // Returns the current playback position in seconds.
michael@0:   // Called from the main thread to get the current frame time. The decoder
michael@0:   // monitor must be obtained before calling this.
michael@0:   double GetCurrentTime() const;
michael@0: 
michael@0:   // Clear the flag indicating that a playback position change event
michael@0:   // is currently queued. This is called from the main thread and must
michael@0:   // be called with the decode monitor held.
michael@0:   void ClearPositionChangeFlag();
michael@0: 
michael@0:   // Called from the main thread or the decoder thread to set whether the media
michael@0:   // resource can seek into unbuffered ranges. The decoder monitor must be
michael@0:   // obtained before calling this.
michael@0:   void SetTransportSeekable(bool aSeekable);
michael@0: 
michael@0:   // Called from the main thread or the decoder thread to set whether the media
michael@0:   // can seek to random location. This is not true for chained ogg and WebM
michael@0:   // media without index. The decoder monitor must be obtained before calling
michael@0:   // this.
michael@0:   void SetMediaSeekable(bool aSeekable);
michael@0: 
michael@0:   // Update the playback position. This can result in a timeupdate event
michael@0:   // and an invalidate of the frame being dispatched asynchronously if
michael@0:   // there is no such event currently queued.
michael@0:   // Only called on the decoder thread. Must be called with
michael@0:   // the decode monitor held.
michael@0:   void UpdatePlaybackPosition(int64_t aTime);
michael@0: 
michael@0:   // Causes the state machine to switch to buffering state, and to
michael@0:   // immediately stop playback and buffer downloaded data. Must be called
michael@0:   // with the decode monitor held. Called on the state machine thread and
michael@0:   // the main thread.
michael@0:   void StartBuffering();
michael@0: 
michael@0:   // This is called on the state machine thread and audio thread.
michael@0:   // The decoder monitor must be obtained before calling this.
michael@0:   bool HasAudio() const {
michael@0:     AssertCurrentThreadInMonitor();
michael@0:     return mInfo.HasAudio();
michael@0:   }
michael@0: 
michael@0:   // This is called on the state machine thread and audio thread.
michael@0:   // The decoder monitor must be obtained before calling this.
michael@0:   bool HasVideo() const {
michael@0:     AssertCurrentThreadInMonitor();
michael@0:     return mInfo.HasVideo();
michael@0:   }
michael@0: 
michael@0:   // Should be called by main thread.
michael@0:   bool HaveNextFrameData();
michael@0: 
michael@0:   // Must be called with the decode monitor held.
michael@0:   bool IsBuffering() const {
michael@0:     AssertCurrentThreadInMonitor();
michael@0: 
michael@0:     return mState == DECODER_STATE_BUFFERING;
michael@0:   }
michael@0: 
michael@0:   // Must be called with the decode monitor held.
michael@0:   bool IsSeeking() const {
michael@0:     AssertCurrentThreadInMonitor();
michael@0: 
michael@0:     return mState == DECODER_STATE_SEEKING;
michael@0:   }
michael@0: 
michael@0:   nsresult GetBuffered(dom::TimeRanges* aBuffered);
michael@0: 
michael@0:   void SetPlaybackRate(double aPlaybackRate);
michael@0:   void SetPreservesPitch(bool aPreservesPitch);
michael@0: 
michael@0:   size_t SizeOfVideoQueue() {
michael@0:     if (mReader) {
michael@0:       return mReader->SizeOfVideoQueueInBytes();
michael@0:     }
michael@0:     return 0;
michael@0:   }
michael@0: 
michael@0:   size_t SizeOfAudioQueue() {
michael@0:     if (mReader) {
michael@0:       return mReader->SizeOfAudioQueueInBytes();
michael@0:     }
michael@0:     return 0;
michael@0:   }
michael@0: 
michael@0:   void NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64_t aOffset);
michael@0: 
michael@0:   int64_t GetEndMediaTime() const {
michael@0:     AssertCurrentThreadInMonitor();
michael@0:     return mEndTime;
michael@0:   }
michael@0: 
michael@0:   bool IsTransportSeekable() {
michael@0:     AssertCurrentThreadInMonitor();
michael@0:     return mTransportSeekable;
michael@0:   }
michael@0: 
michael@0:   bool IsMediaSeekable() {
michael@0:     AssertCurrentThreadInMonitor();
michael@0:     return mMediaSeekable;
michael@0:   }
michael@0: 
michael@0:   // Returns the shared state machine thread.
michael@0:   nsIEventTarget* GetStateMachineThread();
michael@0: 
michael@0:   // Calls ScheduleStateMachine() after taking the decoder lock. Also
michael@0:   // notifies the decoder thread in case it's waiting on the decoder lock.
michael@0:   void ScheduleStateMachineWithLockAndWakeDecoder();
michael@0: 
michael@0:   // Schedules the shared state machine thread to run the state machine
michael@0:   // in aUsecs microseconds from now, if it's not already scheduled to run
michael@0:   // earlier, in which case the request is discarded.
michael@0:   nsresult ScheduleStateMachine(int64_t aUsecs = 0);
michael@0: 
michael@0:   // Timer function to implement ScheduleStateMachine(aUsecs).
michael@0:   nsresult TimeoutExpired(int aGeneration);
michael@0: 
michael@0:   // Set the media fragment end time. aEndTime is in microseconds.
michael@0:   void SetFragmentEndTime(int64_t aEndTime);
michael@0: 
michael@0:   // Drop reference to decoder.  Only called during shutdown dance.
michael@0:   void ReleaseDecoder() {
michael@0:     MOZ_ASSERT(mReader);
michael@0:     if (mReader) {
michael@0:       mReader->ReleaseDecoder();
michael@0:     }
michael@0:     mDecoder = nullptr;
michael@0:   }
michael@0: 
michael@0:   // If we're playing into a MediaStream, record the current point in the
michael@0:   // MediaStream and the current point in our media resource so later we can
michael@0:   // convert MediaStream playback positions to media resource positions. Best to
michael@0:   // call this while we're not playing (while the MediaStream is blocked). Can
michael@0:   // be called on any thread with the decoder monitor held.
michael@0:   void SetSyncPointForMediaStream();
michael@0:   int64_t GetCurrentTimeViaMediaStreamSync();
michael@0: 
michael@0:   // Copy queued audio/video data in the reader to any output MediaStreams that
michael@0:   // need it.
michael@0:   void SendStreamData();
michael@0:   void FinishStreamData();
michael@0:   bool HaveEnoughDecodedAudio(int64_t aAmpleAudioUSecs);
michael@0:   bool HaveEnoughDecodedVideo();
michael@0: 
michael@0:   // Returns true if the state machine has shutdown or is in the process of
michael@0:   // shutting down. The decoder monitor must be held while calling this.
michael@0:   bool IsShutdown();
michael@0: 
michael@0:   void QueueMetadata(int64_t aPublishTime, int aChannels, int aRate, bool aHasAudio, bool aHasVideo, MetadataTags* aTags);
michael@0: 
michael@0:   // Returns true if we're currently playing. The decoder monitor must
michael@0:   // be held.
michael@0:   bool IsPlaying();
michael@0: 
michael@0:   // Called when the reader may have acquired the hardware resources required
michael@0:   // to begin decoding. The state machine may move into DECODING_METADATA if
michael@0:   // appropriate. The decoder monitor must be held while calling this.
michael@0:   void NotifyWaitingForResourcesStatusChanged();
michael@0: 
michael@0:   // Notifies the state machine that should minimize the number of samples
michael@0:   // decoded we preroll, until playback starts. The first time playback starts
michael@0:   // the state machine is free to return to prerolling normally. Note
michael@0:   // "prerolling" in this context refers to when we decode and buffer decoded
michael@0:   // samples in advance of when they're needed for playback.
michael@0:   void SetMinimizePrerollUntilPlaybackStarts();
michael@0: 
michael@0: protected:
michael@0:   virtual ~MediaDecoderStateMachine();
michael@0: 
michael@0:   void AssertCurrentThreadInMonitor() const { mDecoder->GetReentrantMonitor().AssertCurrentThreadIn(); }
michael@0: 
michael@0:   class WakeDecoderRunnable : public nsRunnable {
michael@0:   public:
michael@0:     WakeDecoderRunnable(MediaDecoderStateMachine* aSM)
michael@0:       : mMutex("WakeDecoderRunnable"), mStateMachine(aSM) {}
michael@0:     NS_IMETHOD Run() MOZ_OVERRIDE
michael@0:     {
michael@0:       nsRefPtr<MediaDecoderStateMachine> stateMachine;
michael@0:       {
michael@0:         // Don't let Run() (called by media stream graph thread) race with
michael@0:         // Revoke() (called by decoder state machine thread)
michael@0:         MutexAutoLock lock(mMutex);
michael@0:         if (!mStateMachine)
michael@0:           return NS_OK;
michael@0:         stateMachine = mStateMachine;
michael@0:       }
michael@0:       stateMachine->ScheduleStateMachineWithLockAndWakeDecoder();
michael@0:       return NS_OK;
michael@0:     }
michael@0:     void Revoke()
michael@0:     {
michael@0:       MutexAutoLock lock(mMutex);
michael@0:       mStateMachine = nullptr;
michael@0:     }
michael@0: 
michael@0:     Mutex mMutex;
michael@0:     // Protected by mMutex.
michael@0:     // We don't use an owning pointer here, because keeping mStateMachine alive
michael@0:     // would mean in some cases we'd have to destroy mStateMachine from this
michael@0:     // object, which would be problematic since MediaDecoderStateMachine can
michael@0:     // only be destroyed on the main thread whereas this object can be destroyed
michael@0:     // on the media stream graph thread.
michael@0:     MediaDecoderStateMachine* mStateMachine;
michael@0:   };
michael@0:   WakeDecoderRunnable* GetWakeDecoderRunnable();
michael@0: 
michael@0:   MediaQueue<AudioData>& AudioQueue() { return mReader->AudioQueue(); }
michael@0:   MediaQueue<VideoData>& VideoQueue() { return mReader->VideoQueue(); }
michael@0: 
michael@0:   // True if our buffers of decoded audio are not full, and we should
michael@0:   // decode more.
michael@0:   bool NeedToDecodeAudio();
michael@0: 
michael@0:   // Decodes some audio. This should be run on the decode task queue.
michael@0:   void DecodeAudio();
michael@0: 
michael@0:   // True if our buffers of decoded video are not full, and we should
michael@0:   // decode more.
michael@0:   bool NeedToDecodeVideo();
michael@0: 
michael@0:   // Decodes some video. This should be run on the decode task queue.
michael@0:   void DecodeVideo();
michael@0: 
michael@0:   // Returns true if we've got less than aAudioUsecs microseconds of decoded
michael@0:   // and playable data. The decoder monitor must be held.
michael@0:   bool HasLowDecodedData(int64_t aAudioUsecs);
michael@0: 
michael@0:   // Returns true if we're running low on data which is not yet decoded.
michael@0:   // The decoder monitor must be held.
michael@0:   bool HasLowUndecodedData();
michael@0: 
michael@0:   // Returns true if we have less than aUsecs of undecoded data available.
michael@0:   bool HasLowUndecodedData(double aUsecs);
michael@0: 
michael@0:   // Returns the number of unplayed usecs of audio we've got decoded and/or
michael@0:   // pushed to the hardware waiting to play. This is how much audio we can
michael@0:   // play without having to run the audio decoder. The decoder monitor
michael@0:   // must be held.
michael@0:   int64_t AudioDecodedUsecs();
michael@0: 
michael@0:   // Returns true when there's decoded audio waiting to play.
michael@0:   // The decoder monitor must be held.
michael@0:   bool HasFutureAudio();
michael@0: 
michael@0:   // Returns true if we recently exited "quick buffering" mode.
michael@0:   bool JustExitedQuickBuffering();
michael@0: 
michael@0:   // Waits on the decoder ReentrantMonitor for aUsecs microseconds. If the decoder
michael@0:   // monitor is awoken by a Notify() call, we'll continue waiting, unless
michael@0:   // we've moved into shutdown state. This enables us to ensure that we
michael@0:   // wait for a specified time, and that the myriad of Notify()s we do on
michael@0:   // the decoder monitor don't cause the audio thread to be starved. aUsecs
michael@0:   // values of less than 1 millisecond are rounded up to 1 millisecond
michael@0:   // (see bug 651023). The decoder monitor must be held. Called only on the
michael@0:   // audio thread.
michael@0:   void Wait(int64_t aUsecs);
michael@0: 
michael@0:   // Dispatches an asynchronous event to update the media element's ready state.
michael@0:   void UpdateReadyState();
michael@0: 
michael@0:   // Resets playback timing data. Called when we seek, on the decode thread.
michael@0:   void ResetPlayback();
michael@0: 
michael@0:   // Returns the audio clock, if we have audio, or -1 if we don't.
michael@0:   // Called on the state machine thread.
michael@0:   int64_t GetAudioClock();
michael@0: 
michael@0:   // Get the video stream position, taking the |playbackRate| change into
michael@0:   // account. This is a position in the media, not the duration of the playback
michael@0:   // so far.
michael@0:   int64_t GetVideoStreamPosition();
michael@0: 
michael@0:   // Return the current time, either the audio clock if available (if the media
michael@0:   // has audio, and the playback is possible), or a clock for the video.
michael@0:   // Called on the state machine thread.
michael@0:   int64_t GetClock();
michael@0: 
michael@0:   // Returns the presentation time of the first audio or video frame in the
michael@0:   // media.  If the media has video, it returns the first video frame. The
michael@0:   // decoder monitor must be held with exactly one lock count. Called on the
michael@0:   // state machine thread.
michael@0:   VideoData* FindStartTime();
michael@0: 
michael@0:   // Update only the state machine's current playback position (and duration,
michael@0:   // if unknown).  Does not update the playback position on the decoder or
michael@0:   // media element -- use UpdatePlaybackPosition for that.  Called on the state
michael@0:   // machine thread, caller must hold the decoder lock.
michael@0:   void UpdatePlaybackPositionInternal(int64_t aTime);
michael@0: 
michael@0:   // Pushes the image down the rendering pipeline. Called on the shared state
michael@0:   // machine thread. The decoder monitor must *not* be held when calling this.
michael@0:   void RenderVideoFrame(VideoData* aData, TimeStamp aTarget);
michael@0: 
michael@0:   // If we have video, display a video frame if it's time for display has
michael@0:   // arrived, otherwise sleep until it's time for the next frame. Update the
michael@0:   // current frame time as appropriate, and trigger ready state update.  The
michael@0:   // decoder monitor must be held with exactly one lock count. Called on the
michael@0:   // state machine thread.
michael@0:   void AdvanceFrame();
michael@0: 
michael@0:   // Write aFrames of audio frames of silence to the audio hardware. Returns
michael@0:   // the number of frames actually written. The write size is capped at
michael@0:   // SILENCE_BYTES_CHUNK (32kB), so must be called in a loop to write the
michael@0:   // desired number of frames. This ensures that the playback position
michael@0:   // advances smoothly, and guarantees that we don't try to allocate an
michael@0:   // impossibly large chunk of memory in order to play back silence. Called
michael@0:   // on the audio thread.
michael@0:   uint32_t PlaySilence(uint32_t aFrames,
michael@0:                        uint32_t aChannels,
michael@0:                        uint64_t aFrameOffset);
michael@0: 
michael@0:   // Pops an audio chunk from the front of the audio queue, and pushes its
michael@0:   // audio data to the audio hardware.
michael@0:   uint32_t PlayFromAudioQueue(uint64_t aFrameOffset, uint32_t aChannels);
michael@0: 
michael@0:   // Stops the audio thread. The decoder monitor must be held with exactly
michael@0:   // one lock count. Called on the state machine thread.
michael@0:   void StopAudioThread();
michael@0: 
michael@0:   // Starts the audio thread. The decoder monitor must be held with exactly
michael@0:   // one lock count. Called on the state machine thread.
michael@0:   nsresult StartAudioThread();
michael@0: 
michael@0:   // The main loop for the audio thread. Sent to the thread as
michael@0:   // an nsRunnableMethod. This continually does blocking writes to
michael@0:   // to audio stream to play audio data.
michael@0:   void AudioLoop();
michael@0: 
michael@0:   // Sets internal state which causes playback of media to pause.
michael@0:   // The decoder monitor must be held.
michael@0:   void StopPlayback();
michael@0: 
michael@0:   // Sets internal state which causes playback of media to begin or resume.
michael@0:   // Must be called with the decode monitor held.
michael@0:   void StartPlayback();
michael@0: 
michael@0:   // Moves the decoder into decoding state. Called on the state machine
michael@0:   // thread. The decoder monitor must be held.
michael@0:   void StartDecoding();
michael@0: 
michael@0:   // Moves the decoder into the shutdown state, and dispatches an error
michael@0:   // event to the media element. This begins shutting down the decoder.
michael@0:   // The decoder monitor must be held. This is only called on the
michael@0:   // decode thread.
michael@0:   void DecodeError();
michael@0: 
michael@0:   void StartWaitForResources();
michael@0: 
michael@0:   // Dispatches a task to the decode task queue to begin decoding metadata.
michael@0:   // This is threadsafe and can be called on any thread.
michael@0:   // The decoder monitor must be held.
michael@0:   nsresult EnqueueDecodeMetadataTask();
michael@0: 
michael@0:   nsresult DispatchAudioDecodeTaskIfNeeded();
michael@0: 
michael@0:   // Ensures a to decode audio has been dispatched to the decode task queue.
michael@0:   // If a task to decode has already been dispatched, this does nothing,
michael@0:   // otherwise this dispatches a task to do the decode.
michael@0:   // This is called on the state machine or decode threads.
michael@0:   // The decoder monitor must be held.
michael@0:   nsresult EnsureAudioDecodeTaskQueued();
michael@0: 
michael@0:   nsresult DispatchVideoDecodeTaskIfNeeded();
michael@0: 
michael@0:   // Ensures a to decode video has been dispatched to the decode task queue.
michael@0:   // If a task to decode has already been dispatched, this does nothing,
michael@0:   // otherwise this dispatches a task to do the decode.
michael@0:   // The decoder monitor must be held.
michael@0:   nsresult EnsureVideoDecodeTaskQueued();
michael@0: 
michael@0:   // Dispatches a task to the decode task queue to seek the decoder.
michael@0:   // The decoder monitor must be held.
michael@0:   nsresult EnqueueDecodeSeekTask();
michael@0: 
michael@0:   // Calls the reader's SetIdle(), with aIsIdle as parameter. This is only
michael@0:   // called in a task dispatched to the decode task queue, don't call it
michael@0:   // directly.
michael@0:   void SetReaderIdle();
michael@0:   void SetReaderActive();
michael@0: 
michael@0:   // Re-evaluates the state and determines whether we need to dispatch
michael@0:   // events to run the decode, or if not whether we should set the reader
michael@0:   // to idle mode. This is threadsafe, and can be called from any thread.
michael@0:   // The decoder monitor must be held.
michael@0:   void DispatchDecodeTasksIfNeeded();
michael@0: 
michael@0:   // Called before we do anything on the decode task queue to set the reader
michael@0:   // as not idle if it was idle. This is called before we decode, seek, or
michael@0:   // decode metadata (in case we were dormant or awaiting resources).
michael@0:   void EnsureActive();
michael@0: 
michael@0:   // Queries our state to see whether the decode has finished for all streams.
michael@0:   // If so, we move into DECODER_STATE_COMPLETED and schedule the state machine
michael@0:   // to run.
michael@0:   // The decoder monitor must be held.
michael@0:   void CheckIfDecodeComplete();
michael@0: 
michael@0:   // Returns the "media time". This is the absolute time which the media
michael@0:   // playback has reached. i.e. this returns values in the range
michael@0:   // [mStartTime, mEndTime], and mStartTime will not be 0 if the media does
michael@0:   // not start at 0. Note this is different to the value returned
michael@0:   // by GetCurrentTime(), which is in the range [0,duration].
michael@0:   int64_t GetMediaTime() const {
michael@0:     AssertCurrentThreadInMonitor();
michael@0:     return mStartTime + mCurrentFrameTime;
michael@0:   }
michael@0: 
michael@0:   // Returns an upper bound on the number of microseconds of audio that is
michael@0:   // decoded and playable. This is the sum of the number of usecs of audio which
michael@0:   // is decoded and in the reader's audio queue, and the usecs of unplayed audio
michael@0:   // which has been pushed to the audio hardware for playback. Note that after
michael@0:   // calling this, the audio hardware may play some of the audio pushed to
michael@0:   // hardware, so this can only be used as a upper bound. The decoder monitor
michael@0:   // must be held when calling this. Called on the decode thread.
michael@0:   int64_t GetDecodedAudioDuration();
michael@0: 
michael@0:   // Load metadata. Called on the decode thread. The decoder monitor
michael@0:   // must be held with exactly one lock count.
michael@0:   nsresult DecodeMetadata();
michael@0: 
michael@0:   // Seeks to mSeekTarget. Called on the decode thread. The decoder monitor
michael@0:   // must be held with exactly one lock count.
michael@0:   void DecodeSeek();
michael@0: 
michael@0:   // Decode loop, decodes data until EOF or shutdown.
michael@0:   // Called on the decode thread.
michael@0:   void DecodeLoop();
michael@0: 
michael@0:   void CallDecodeMetadata();
michael@0: 
michael@0:   // Copy audio from an AudioData packet to aOutput. This may require
michael@0:   // inserting silence depending on the timing of the audio packet.
michael@0:   void SendStreamAudio(AudioData* aAudio, DecodedStreamData* aStream,
michael@0:                        AudioSegment* aOutput);
michael@0: 
michael@0:   // State machine thread run function. Defers to RunStateMachine().
michael@0:   nsresult CallRunStateMachine();
michael@0: 
michael@0:   // Performs one "cycle" of the state machine. Polls the state, and may send
michael@0:   // a video frame to be displayed, and generally manages the decode. Called
michael@0:   // periodically via timer to ensure the video stays in sync.
michael@0:   nsresult RunStateMachine();
michael@0: 
michael@0:   bool IsStateMachineScheduled() const {
michael@0:     AssertCurrentThreadInMonitor();
michael@0:     return !mTimeout.IsNull();
michael@0:   }
michael@0: 
michael@0:   // Returns true if we're not playing and the decode thread has filled its
michael@0:   // decode buffers and is waiting. We can shut the decode thread down in this
michael@0:   // case as it may not be needed again.
michael@0:   bool IsPausedAndDecoderWaiting();
michael@0: 
michael@0:   // The decoder object that created this state machine. The state machine
michael@0:   // holds a strong reference to the decoder to ensure that the decoder stays
michael@0:   // alive once media element has started the decoder shutdown process, and has
michael@0:   // dropped its reference to the decoder. This enables the state machine to
michael@0:   // keep using the decoder's monitor until the state machine has finished
michael@0:   // shutting down, without fear of the monitor being destroyed. After
michael@0:   // shutting down, the state machine will then release this reference,
michael@0:   // causing the decoder to be destroyed. This is accessed on the decode,
michael@0:   // state machine, audio and main threads.
michael@0:   nsRefPtr<MediaDecoder> mDecoder;
michael@0: 
michael@0:   // The decoder monitor must be obtained before modifying this state.
michael@0:   // NotifyAll on the monitor must be called when the state is changed so
michael@0:   // that interested threads can wake up and alter behaviour if appropriate
michael@0:   // Accessed on state machine, audio, main, and AV thread.
michael@0:   State mState;
michael@0: 
michael@0:   // Thread for pushing audio onto the audio hardware.
michael@0:   // The "audio push thread".
michael@0:   nsCOMPtr<nsIThread> mAudioThread;
michael@0: 
michael@0:   // The task queue in which we run decode tasks. This is referred to as
michael@0:   // the "decode thread", though in practise tasks can run on a different
michael@0:   // thread every time they're called.
michael@0:   RefPtr<MediaTaskQueue> mDecodeTaskQueue;
michael@0: 
michael@0:   RefPtr<SharedThreadPool> mStateMachineThreadPool;
michael@0: 
michael@0:   // Timer to run the state machine cycles. Used by
michael@0:   // ScheduleStateMachine(). Access protected by decoder monitor.
michael@0:   nsCOMPtr<nsITimer> mTimer;
michael@0: 
michael@0:   // Timestamp at which the next state machine cycle will run.
michael@0:   // Access protected by decoder monitor.
michael@0:   TimeStamp mTimeout;
michael@0: 
michael@0:   // Used to check if there are state machine cycles are running in sequence.
michael@0:   DebugOnly<bool> mInRunningStateMachine;
michael@0: 
michael@0:   // The time that playback started from the system clock. This is used for
michael@0:   // timing the presentation of video frames when there's no audio.
michael@0:   // Accessed only via the state machine thread.
michael@0:   TimeStamp mPlayStartTime;
michael@0: 
michael@0:   // When we start writing decoded data to a new DecodedDataStream, or we
michael@0:   // restart writing due to PlaybackStarted(), we record where we are in the
michael@0:   // MediaStream and what that corresponds to in the media.
michael@0:   StreamTime mSyncPointInMediaStream;
michael@0:   int64_t mSyncPointInDecodedStream; // microseconds
michael@0: 
michael@0:   // When the playbackRate changes, and there is no audio clock, it is necessary
michael@0:   // to reset the mPlayStartTime. This is done next time the clock is queried,
michael@0:   // when this member is true. Access protected by decoder monitor.
michael@0:   bool mResetPlayStartTime;
michael@0: 
michael@0:   // The amount of time we've spent playing already the media. The current
michael@0:   // playback position is therefore |Now() - mPlayStartTime +
michael@0:   // mPlayDuration|, which must be adjusted by mStartTime if used with media
michael@0:   // timestamps.  Accessed only via the state machine thread.
michael@0:   int64_t mPlayDuration;
michael@0: 
michael@0:   // Time that buffering started. Used for buffering timeout and only
michael@0:   // accessed on the state machine thread. This is null while we're not
michael@0:   // buffering.
michael@0:   TimeStamp mBufferingStart;
michael@0: 
michael@0:   // Start time of the media, in microseconds. This is the presentation
michael@0:   // time of the first frame decoded from the media, and is used to calculate
michael@0:   // duration and as a bounds for seeking. Accessed on state machine, decode,
michael@0:   // and main threads. Access controlled by decoder monitor.
michael@0:   int64_t mStartTime;
michael@0: 
michael@0:   // Time of the last frame in the media, in microseconds. This is the
michael@0:   // end time of the last frame in the media. Accessed on state
michael@0:   // machine, decode, and main threads. Access controlled by decoder monitor.
michael@0:   int64_t mEndTime;
michael@0: 
michael@0:   // Position to seek to in microseconds when the seek state transition occurs.
michael@0:   // The decoder monitor lock must be obtained before reading or writing
michael@0:   // this value. Accessed on main and decode thread.
michael@0:   SeekTarget mSeekTarget;
michael@0: 
michael@0:   // Media Fragment end time in microseconds. Access controlled by decoder monitor.
michael@0:   int64_t mFragmentEndTime;
michael@0: 
michael@0:   // The audio stream resource. Used on the state machine, and audio threads.
michael@0:   // This is created and destroyed on the audio thread, while holding the
michael@0:   // decoder monitor, so if this is used off the audio thread, you must
michael@0:   // first acquire the decoder monitor and check that it is non-null.
michael@0:   RefPtr<AudioStream> mAudioStream;
michael@0: 
michael@0:   // The reader, don't call its methods with the decoder monitor held.
michael@0:   // This is created in the play state machine's constructor, and destroyed
michael@0:   // in the play state machine's destructor.
michael@0:   nsAutoPtr<MediaDecoderReader> mReader;
michael@0: 
michael@0:   // Accessed only on the state machine thread.
michael@0:   // Not an nsRevocableEventPtr since we must Revoke() it well before
michael@0:   // this object is destroyed, anyway.
michael@0:   // Protected by decoder monitor except during the SHUTDOWN state after the
michael@0:   // decoder thread has been stopped.
michael@0:   nsRevocableEventPtr<WakeDecoderRunnable> mPendingWakeDecoder;
michael@0: 
michael@0:   // The time of the current frame in microseconds. This is referenced from
michael@0:   // 0 which is the initial playback position. Set by the state machine
michael@0:   // thread, and read-only from the main thread to get the current
michael@0:   // time value. Synchronised via decoder monitor.
michael@0:   int64_t mCurrentFrameTime;
michael@0: 
michael@0:   // The presentation time of the first audio frame that was played in
michael@0:   // microseconds. We can add this to the audio stream position to determine
michael@0:   // the current audio time. Accessed on audio and state machine thread.
michael@0:   // Synchronized by decoder monitor.
michael@0:   int64_t mAudioStartTime;
michael@0: 
michael@0:   // The end time of the last audio frame that's been pushed onto the audio
michael@0:   // hardware in microseconds. This will approximately be the end time of the
michael@0:   // audio stream, unless another frame is pushed to the hardware.
michael@0:   int64_t mAudioEndTime;
michael@0: 
michael@0:   // The presentation end time of the last video frame which has been displayed
michael@0:   // in microseconds. Accessed from the state machine thread.
michael@0:   int64_t mVideoFrameEndTime;
michael@0: 
michael@0:   // Volume of playback. 0.0 = muted. 1.0 = full volume. Read/Written
michael@0:   // from the state machine and main threads. Synchronised via decoder
michael@0:   // monitor.
michael@0:   double mVolume;
michael@0: 
michael@0:   // Playback rate. 1.0 : normal speed, 0.5 : two times slower. Synchronized via
michael@0:   // decoder monitor.
michael@0:   double mPlaybackRate;
michael@0: 
michael@0:   // Pitch preservation for the playback rate. Synchronized via decoder monitor.
michael@0:   bool mPreservesPitch;
michael@0: 
michael@0:   // Position at which the last playback rate change occured, used to compute
michael@0:   // the actual position in the stream when the playback rate changes and there
michael@0:   // is no audio to be sync-ed to. Synchronized via decoder monitor.
michael@0:   int64_t mBasePosition;
michael@0: 
michael@0:   // Time at which we started decoding. Synchronised via decoder monitor.
michael@0:   TimeStamp mDecodeStartTime;
michael@0: 
michael@0:   // The maximum number of second we spend buffering when we are short on
michael@0:   // unbuffered data.
michael@0:   uint32_t mBufferingWait;
michael@0:   int64_t  mLowDataThresholdUsecs;
michael@0: 
michael@0:   // If we've got more than mAmpleVideoFrames decoded video frames waiting in
michael@0:   // the video queue, we will not decode any more video frames until some have
michael@0:   // been consumed by the play state machine thread.
michael@0:   uint32_t mAmpleVideoFrames;
michael@0: 
michael@0:   // Low audio threshold. If we've decoded less than this much audio we
michael@0:   // consider our audio decode "behind", and we may skip video decoding
michael@0:   // in order to allow our audio decoding to catch up. We favour audio
michael@0:   // decoding over video. We increase this threshold if we're slow to
michael@0:   // decode video frames, in order to reduce the chance of audio underruns.
michael@0:   // Note that we don't ever reset this threshold, it only ever grows as
michael@0:   // we detect that the decode can't keep up with rendering.
michael@0:   int64_t mLowAudioThresholdUsecs;
michael@0: 
michael@0:   // Our "ample" audio threshold. Once we've this much audio decoded, we
michael@0:   // pause decoding. If we increase mLowAudioThresholdUsecs, we'll also
michael@0:   // increase this too appropriately (we don't want mLowAudioThresholdUsecs
michael@0:   // to be greater than ampleAudioThreshold, else we'd stop decoding!).
michael@0:   // Note that we don't ever reset this threshold, it only ever grows as
michael@0:   // we detect that the decode can't keep up with rendering.
michael@0:   int64_t mAmpleAudioThresholdUsecs;
michael@0: 
michael@0:   // At the start of decoding we want to "preroll" the decode until we've
michael@0:   // got a few frames decoded before we consider whether decode is falling
michael@0:   // behind. Otherwise our "we're falling behind" logic will trigger
michael@0:   // unneccessarily if we start playing as soon as the first sample is
michael@0:   // decoded. These two fields store how many video frames and audio
michael@0:   // samples we must consume before are considered to be finished prerolling.
michael@0:   uint32_t mAudioPrerollUsecs;
michael@0:   uint32_t mVideoPrerollFrames;
michael@0: 
michael@0:   // When we start decoding (either for the first time, or after a pause)
michael@0:   // we may be low on decoded data. We don't want our "low data" logic to
michael@0:   // kick in and decide that we're low on decoded data because the download
michael@0:   // can't keep up with the decode, and cause us to pause playback. So we
michael@0:   // have a "preroll" stage, where we ignore the results of our "low data"
michael@0:   // logic during the first few frames of our decode. This occurs during
michael@0:   // playback. The flags below are true when the corresponding stream is
michael@0:   // being "prerolled".
michael@0:   bool mIsAudioPrerolling;
michael@0:   bool mIsVideoPrerolling;
michael@0: 
michael@0:   // True when we have an audio stream that we're decoding, and we have not
michael@0:   // yet decoded to end of stream.
michael@0:   bool mIsAudioDecoding;
michael@0: 
michael@0:   // True when we have a video stream that we're decoding, and we have not
michael@0:   // yet decoded to end of stream.
michael@0:   bool mIsVideoDecoding;
michael@0: 
michael@0:   // True when we have dispatched a task to the decode task queue to run
michael@0:   // the audio decode.
michael@0:   bool mDispatchedAudioDecodeTask;
michael@0: 
michael@0:   // True when we have dispatched a task to the decode task queue to run
michael@0:   // the video decode.
michael@0:   bool mDispatchedVideoDecodeTask;
michael@0: 
michael@0:   // True when the reader is initialized, but has been ordered "idle" by the
michael@0:   // state machine. This happens when the MediaQueue's of decoded data are
michael@0:   // "full" and playback is paused. The reader may choose to use the idle
michael@0:   // notification to enter a low power state.
michael@0:   bool mIsReaderIdle;
michael@0: 
michael@0:   // If the video decode is falling behind the audio, we'll start dropping the
michael@0:   // inter-frames up until the next keyframe which is at or before the current
michael@0:   // playback position. skipToNextKeyframe is true if we're currently
michael@0:   // skipping up to the next keyframe.
michael@0:   bool mSkipToNextKeyFrame;
michael@0: 
michael@0:   // True if we shouldn't play our audio (but still write it to any capturing
michael@0:   // streams). When this is true, mStopAudioThread is always true and
michael@0:   // the audio thread will never start again after it has stopped.
michael@0:   bool mAudioCaptured;
michael@0: 
michael@0:   // True if the media resource can be seeked on a transport level. Accessed
michael@0:   // from the state machine and main threads. Synchronised via decoder monitor.
michael@0:   bool mTransportSeekable;
michael@0: 
michael@0:   // True if the media can be seeked. Accessed from the state machine and main
michael@0:   // threads. Synchronised via decoder monitor.
michael@0:   bool mMediaSeekable;
michael@0: 
michael@0:   // True if an event to notify about a change in the playback
michael@0:   // position has been queued, but not yet run. It is set to false when
michael@0:   // the event is run. This allows coalescing of these events as they can be
michael@0:   // produced many times per second. Synchronised via decoder monitor.
michael@0:   // Accessed on main and state machine threads.
michael@0:   bool mPositionChangeQueued;
michael@0: 
michael@0:   // True if the audio playback thread has finished. It is finished
michael@0:   // when either all the audio frames have completed playing, or we've moved
michael@0:   // into shutdown state, and the threads are to be
michael@0:   // destroyed. Written by the audio playback thread and read and written by
michael@0:   // the state machine thread. Synchronised via decoder monitor.
michael@0:   // When data is being sent to a MediaStream, this is true when all data has
michael@0:   // been written to the MediaStream.
michael@0:   bool mAudioCompleted;
michael@0: 
michael@0:   // True if mDuration has a value obtained from an HTTP header, or from
michael@0:   // the media index/metadata. Accessed on the state machine thread.
michael@0:   bool mGotDurationFromMetaData;
michael@0: 
michael@0:   // True if we've dispatched an event to the decode task queue to call
michael@0:   // DecodeThreadRun(). We use this flag to prevent us from dispatching
michael@0:   // unneccessary runnables, since the decode thread runs in a loop.
michael@0:   bool mDispatchedEventToDecode;
michael@0: 
michael@0:   // False while audio thread should be running. Accessed state machine
michael@0:   // and audio threads. Syncrhonised by decoder monitor.
michael@0:   bool mStopAudioThread;
michael@0: 
michael@0:   // If this is true while we're in buffering mode, we can exit early,
michael@0:   // as it's likely we may be able to playback. This happens when we enter
michael@0:   // buffering mode soon after the decode starts, because the decode-ahead
michael@0:   // ran fast enough to exhaust all data while the download is starting up.
michael@0:   // Synchronised via decoder monitor.
michael@0:   bool mQuickBuffering;
michael@0: 
michael@0:   // True if we should not decode/preroll unnecessary samples, unless we're
michael@0:   // played. "Prerolling" in this context refers to when we decode and
michael@0:   // buffer decoded samples in advance of when they're needed for playback.
michael@0:   // This flag is set for preload=metadata media, and means we won't
michael@0:   // decode more than the first video frame and first block of audio samples
michael@0:   // for that media when we startup, or after a seek. When Play() is called,
michael@0:   // we reset this flag, as we assume the user is playing the media, so
michael@0:   // prerolling is appropriate then. This flag is used to reduce the overhead
michael@0:   // of prerolling samples for media elements that may not play, both
michael@0:   // memory and CPU overhead.
michael@0:   bool mMinimizePreroll;
michael@0: 
michael@0:   // True if the decode thread has gone filled its buffers and is now
michael@0:   // waiting to be awakened before it continues decoding. Synchronized
michael@0:   // by the decoder monitor.
michael@0:   bool mDecodeThreadWaiting;
michael@0: 
michael@0:   // True is we are decoding a realtime stream, like a camera stream
michael@0:   bool mRealTime;
michael@0: 
michael@0:   // Stores presentation info required for playback. The decoder monitor
michael@0:   // must be held when accessing this.
michael@0:   MediaInfo mInfo;
michael@0: 
michael@0:   mozilla::MediaMetadataManager mMetadataManager;
michael@0: 
michael@0:   MediaDecoderOwner::NextFrameStatus mLastFrameStatus;
michael@0: 
michael@0:   // The id of timer tasks, used to ignore tasks that are scheduled previously.
michael@0:   int mTimerId;
michael@0: };
michael@0: 
michael@0: } // namespace mozilla;
michael@0: #endif