content/media/wmf/WMFReader.cpp@97036ab72558
content/media/wmf/WMFReader.cpp
Tue, 06 Jan 2015 21:39:09 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Tue, 06 Jan 2015 21:39:09 +0100
- branch
- TOR_BUG_9701
- changeset 8
- 97036ab72558
- permissions
- -rw-r--r--
Conditionally force memory storage according to privacy.thirdparty.isolate;
This solves Tor bug #9701, complying with disk avoidance documented in
https://www.torproject.org/projects/torbrowser/design/#disk-avoidance.
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "WMFReader.h"
8 #include "WMFDecoder.h"
9 #include "WMFUtils.h"
10 #include "WMFByteStream.h"
11 #include "WMFSourceReaderCallback.h"
12 #include "mozilla/ArrayUtils.h"
13 #include "mozilla/dom/TimeRanges.h"
14 #include "mozilla/dom/HTMLMediaElement.h"
15 #include "mozilla/Preferences.h"
16 #include "DXVA2Manager.h"
17 #include "ImageContainer.h"
18 #include "Layers.h"
19 #include "mozilla/layers/LayersTypes.h"
21 #ifndef MOZ_SAMPLE_TYPE_FLOAT32
22 #error We expect 32bit float audio samples on desktop for the Windows Media Foundation media backend.
23 #endif
25 #include "MediaDecoder.h"
26 #include "VideoUtils.h"
27 #include "gfx2DGlue.h"
29 using namespace mozilla::gfx;
30 using mozilla::layers::Image;
31 using mozilla::layers::LayerManager;
32 using mozilla::layers::LayersBackend;
34 namespace mozilla {
36 #ifdef PR_LOGGING
37 extern PRLogModuleInfo* gMediaDecoderLog;
38 #define DECODER_LOG(...) PR_LOG(gMediaDecoderLog, PR_LOG_DEBUG, (__VA_ARGS__))
39 #else
40 #define DECODER_LOG(...)
41 #endif
43 // Uncomment to enable verbose per-sample logging.
44 //#define LOG_SAMPLE_DECODE 1
46 WMFReader::WMFReader(AbstractMediaDecoder* aDecoder)
47 : MediaDecoderReader(aDecoder),
48 mSourceReader(nullptr),
49 mAudioChannels(0),
50 mAudioBytesPerSample(0),
51 mAudioRate(0),
52 mVideoWidth(0),
53 mVideoHeight(0),
54 mVideoStride(0),
55 mAudioFrameSum(0),
56 mAudioFrameOffset(0),
57 mHasAudio(false),
58 mHasVideo(false),
59 mUseHwAccel(false),
60 mMustRecaptureAudioPosition(true),
61 mIsMP3Enabled(WMFDecoder::IsMP3Supported()),
62 mCOMInitialized(false)
63 {
64 NS_ASSERTION(NS_IsMainThread(), "Must be on main thread.");
65 MOZ_COUNT_CTOR(WMFReader);
66 }
68 WMFReader::~WMFReader()
69 {
70 NS_ASSERTION(NS_IsMainThread(), "Must be on main thread.");
72 // Note: We must shutdown the byte stream before calling MFShutdown, else we
73 // get assertion failures when unlocking the byte stream's work queue.
74 if (mByteStream) {
75 DebugOnly<nsresult> rv = mByteStream->Shutdown();
76 NS_ASSERTION(NS_SUCCEEDED(rv), "Failed to shutdown WMFByteStream");
77 }
78 DebugOnly<HRESULT> hr = wmf::MFShutdown();
79 NS_ASSERTION(SUCCEEDED(hr), "MFShutdown failed");
80 MOZ_COUNT_DTOR(WMFReader);
81 }
83 bool
84 WMFReader::InitializeDXVA()
85 {
86 if (!Preferences::GetBool("media.windows-media-foundation.use-dxva", false)) {
87 return false;
88 }
89 MOZ_ASSERT(mDecoder->GetImageContainer());
91 // Extract the layer manager backend type so that we can determine
92 // whether it's worthwhile using DXVA. If we're not running with a D3D
93 // layer manager then the readback of decoded video frames from GPU to
94 // CPU memory grinds painting to a halt, and makes playback performance
95 // *worse*.
96 MediaDecoderOwner* owner = mDecoder->GetOwner();
97 NS_ENSURE_TRUE(owner, false);
99 dom::HTMLMediaElement* element = owner->GetMediaElement();
100 NS_ENSURE_TRUE(element, false);
102 nsRefPtr<LayerManager> layerManager =
103 nsContentUtils::LayerManagerForDocument(element->OwnerDoc());
104 NS_ENSURE_TRUE(layerManager, false);
106 LayersBackend backend = layerManager->GetCompositorBackendType();
107 if (backend != LayersBackend::LAYERS_D3D9 &&
108 backend != LayersBackend::LAYERS_D3D10 &&
109 backend != LayersBackend::LAYERS_D3D11) {
110 return false;
111 }
113 mDXVA2Manager = DXVA2Manager::Create();
115 return mDXVA2Manager != nullptr;
116 }
118 nsresult
119 WMFReader::Init(MediaDecoderReader* aCloneDonor)
120 {
121 NS_ASSERTION(NS_IsMainThread(), "Must be on main thread.");
123 nsresult rv = WMFDecoder::LoadDLLs();
124 NS_ENSURE_SUCCESS(rv, rv);
126 if (FAILED(wmf::MFStartup())) {
127 NS_WARNING("Failed to initialize Windows Media Foundation");
128 return NS_ERROR_FAILURE;
129 }
131 mSourceReaderCallback = new WMFSourceReaderCallback();
133 // Must be created on main thread.
134 mByteStream = new WMFByteStream(mDecoder->GetResource(), mSourceReaderCallback);
135 rv = mByteStream->Init();
136 NS_ENSURE_SUCCESS(rv, rv);
138 if (mDecoder->GetImageContainer() != nullptr &&
139 IsVideoContentType(mDecoder->GetResource()->GetContentType())) {
140 mUseHwAccel = InitializeDXVA();
141 } else {
142 mUseHwAccel = false;
143 }
145 return NS_OK;
146 }
148 bool
149 WMFReader::HasAudio()
150 {
151 NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
152 return mHasAudio;
153 }
155 bool
156 WMFReader::HasVideo()
157 {
158 NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
159 return mHasVideo;
160 }
162 static HRESULT
163 ConfigureSourceReaderStream(IMFSourceReader *aReader,
164 const DWORD aStreamIndex,
165 const GUID& aOutputSubType,
166 const GUID* aAllowedInSubTypes,
167 const uint32_t aNumAllowedInSubTypes)
168 {
169 NS_ENSURE_TRUE(aReader, E_POINTER);
170 NS_ENSURE_TRUE(aAllowedInSubTypes, E_POINTER);
172 RefPtr<IMFMediaType> nativeType;
173 RefPtr<IMFMediaType> type;
174 HRESULT hr;
176 // Find the native format of the stream.
177 hr = aReader->GetNativeMediaType(aStreamIndex, 0, byRef(nativeType));
178 NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
180 // Get the native output subtype of the stream. This denotes the uncompressed
181 // type.
182 GUID subType;
183 hr = nativeType->GetGUID(MF_MT_SUBTYPE, &subType);
184 NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
186 // Ensure the input type of the media is in the allowed formats list.
187 bool isSubTypeAllowed = false;
188 for (uint32_t i = 0; i < aNumAllowedInSubTypes; i++) {
189 if (aAllowedInSubTypes[i] == subType) {
190 isSubTypeAllowed = true;
191 break;
192 }
193 }
194 if (!isSubTypeAllowed) {
195 nsCString name = GetGUIDName(subType);
196 DECODER_LOG("ConfigureSourceReaderStream subType=%s is not allowed to be decoded", name.get());
197 return E_FAIL;
198 }
200 // Find the major type.
201 GUID majorType;
202 hr = nativeType->GetGUID(MF_MT_MAJOR_TYPE, &majorType);
203 NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
205 // Define the output type.
206 hr = wmf::MFCreateMediaType(byRef(type));
207 NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
209 hr = type->SetGUID(MF_MT_MAJOR_TYPE, majorType);
210 NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
212 hr = type->SetGUID(MF_MT_SUBTYPE, aOutputSubType);
213 NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
215 // Set the uncompressed format. This can fail if the decoder can't produce
216 // that type.
217 return aReader->SetCurrentMediaType(aStreamIndex, nullptr, type);
218 }
220 // Returns the duration of the resource, in microseconds.
221 HRESULT
222 GetSourceReaderDuration(IMFSourceReader *aReader,
223 int64_t& aOutDuration)
224 {
225 AutoPropVar var;
226 HRESULT hr = aReader->GetPresentationAttribute(MF_SOURCE_READER_MEDIASOURCE,
227 MF_PD_DURATION,
228 &var);
229 NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
231 // WMF stores duration in hundred nanosecond units.
232 int64_t duration_hns = 0;
233 hr = wmf::PropVariantToInt64(var, &duration_hns);
234 NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
236 aOutDuration = HNsToUsecs(duration_hns);
238 return S_OK;
239 }
241 HRESULT
242 GetSourceReaderCanSeek(IMFSourceReader* aReader, bool& aOutCanSeek)
243 {
244 NS_ENSURE_TRUE(aReader, E_FAIL);
246 HRESULT hr;
247 AutoPropVar var;
248 hr = aReader->GetPresentationAttribute(MF_SOURCE_READER_MEDIASOURCE,
249 MF_SOURCE_READER_MEDIASOURCE_CHARACTERISTICS,
250 &var);
251 NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
253 ULONG flags = 0;
254 hr = wmf::PropVariantToUInt32(var, &flags);
255 NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
257 aOutCanSeek = ((flags & MFMEDIASOURCE_CAN_SEEK) == MFMEDIASOURCE_CAN_SEEK);
259 return S_OK;
260 }
262 HRESULT
263 WMFReader::ConfigureVideoFrameGeometry(IMFMediaType* aMediaType)
264 {
265 NS_ENSURE_TRUE(aMediaType != nullptr, E_POINTER);
266 HRESULT hr;
268 // Verify that the video subtype is what we expect it to be.
269 // When using hardware acceleration/DXVA2 the video format should
270 // be NV12, which is DXVA2's preferred format. For software decoding
271 // we use YV12, as that's easier for us to stick into our rendering
272 // pipeline than NV12. NV12 has interleaved UV samples, whereas YV12
273 // is a planar format.
274 GUID videoFormat;
275 hr = aMediaType->GetGUID(MF_MT_SUBTYPE, &videoFormat);
276 NS_ENSURE_TRUE(videoFormat == MFVideoFormat_NV12 || !mUseHwAccel, E_FAIL);
277 NS_ENSURE_TRUE(videoFormat == MFVideoFormat_YV12 || mUseHwAccel, E_FAIL);
279 nsIntRect pictureRegion;
280 hr = GetPictureRegion(aMediaType, pictureRegion);
281 NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
283 UINT32 width = 0, height = 0;
284 hr = MFGetAttributeSize(aMediaType, MF_MT_FRAME_SIZE, &width, &height);
285 NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
287 uint32_t aspectNum = 0, aspectDenom = 0;
288 hr = MFGetAttributeRatio(aMediaType,
289 MF_MT_PIXEL_ASPECT_RATIO,
290 &aspectNum,
291 &aspectDenom);
292 NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
294 // Calculate and validate the picture region and frame dimensions after
295 // scaling by the pixel aspect ratio.
296 nsIntSize frameSize = nsIntSize(width, height);
297 nsIntSize displaySize = nsIntSize(pictureRegion.width, pictureRegion.height);
298 ScaleDisplayByAspectRatio(displaySize, float(aspectNum) / float(aspectDenom));
299 if (!IsValidVideoRegion(frameSize, pictureRegion, displaySize)) {
300 // Video track's frame sizes will overflow. Ignore the video track.
301 return E_FAIL;
302 }
304 // Success! Save state.
305 mInfo.mVideo.mDisplay = displaySize;
306 GetDefaultStride(aMediaType, &mVideoStride);
307 mVideoWidth = width;
308 mVideoHeight = height;
309 mPictureRegion = pictureRegion;
311 DECODER_LOG("WMFReader frame geometry frame=(%u,%u) stride=%u picture=(%d, %d, %d, %d) display=(%d,%d) PAR=%d:%d",
312 width, height,
313 mVideoStride,
314 mPictureRegion.x, mPictureRegion.y, mPictureRegion.width, mPictureRegion.height,
315 displaySize.width, displaySize.height,
316 aspectNum, aspectDenom);
318 return S_OK;
319 }
321 HRESULT
322 WMFReader::ConfigureVideoDecoder()
323 {
324 NS_ASSERTION(mSourceReader, "Must have a SourceReader before configuring decoders!");
326 // Determine if we have video.
327 if (!mSourceReader ||
328 !SourceReaderHasStream(mSourceReader, MF_SOURCE_READER_FIRST_VIDEO_STREAM)) {
329 // No stream, no error.
330 return S_OK;
331 }
333 if (!mDecoder->GetImageContainer()) {
334 // We can't display the video, so don't bother to decode; disable the stream.
335 return mSourceReader->SetStreamSelection(MF_SOURCE_READER_FIRST_VIDEO_STREAM, FALSE);
336 }
338 static const GUID MP4VideoTypes[] = {
339 MFVideoFormat_H264
340 };
341 HRESULT hr = ConfigureSourceReaderStream(mSourceReader,
342 MF_SOURCE_READER_FIRST_VIDEO_STREAM,
343 mUseHwAccel ? MFVideoFormat_NV12 : MFVideoFormat_YV12,
344 MP4VideoTypes,
345 ArrayLength(MP4VideoTypes));
346 if (FAILED(hr)) {
347 DECODER_LOG("Failed to configured video output");
348 return hr;
349 }
351 RefPtr<IMFMediaType> mediaType;
352 hr = mSourceReader->GetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM,
353 byRef(mediaType));
354 if (FAILED(hr)) {
355 NS_WARNING("Failed to get configured video media type");
356 return hr;
357 }
359 if (FAILED(ConfigureVideoFrameGeometry(mediaType))) {
360 NS_WARNING("Failed configured video frame dimensions");
361 return hr;
362 }
364 DECODER_LOG("Successfully configured video stream");
366 mHasVideo = mInfo.mVideo.mHasVideo = true;
368 return S_OK;
369 }
371 void
372 WMFReader::GetSupportedAudioCodecs(const GUID** aCodecs, uint32_t* aNumCodecs)
373 {
374 MOZ_ASSERT(aCodecs);
375 MOZ_ASSERT(aNumCodecs);
377 if (mIsMP3Enabled) {
378 GUID aacOrMp3 = MFMPEG4Format_Base;
379 aacOrMp3.Data1 = 0x6D703461;// FOURCC('m','p','4','a');
380 static const GUID codecs[] = {
381 MFAudioFormat_AAC,
382 MFAudioFormat_MP3,
383 aacOrMp3
384 };
385 *aCodecs = codecs;
386 *aNumCodecs = ArrayLength(codecs);
387 } else {
388 static const GUID codecs[] = {
389 MFAudioFormat_AAC
390 };
391 *aCodecs = codecs;
392 *aNumCodecs = ArrayLength(codecs);
393 }
394 }
396 HRESULT
397 WMFReader::ConfigureAudioDecoder()
398 {
399 NS_ASSERTION(mSourceReader, "Must have a SourceReader before configuring decoders!");
401 if (!mSourceReader ||
402 !SourceReaderHasStream(mSourceReader, MF_SOURCE_READER_FIRST_AUDIO_STREAM)) {
403 // No stream, no error.
404 return S_OK;
405 }
407 const GUID* codecs;
408 uint32_t numCodecs = 0;
409 GetSupportedAudioCodecs(&codecs, &numCodecs);
411 HRESULT hr = ConfigureSourceReaderStream(mSourceReader,
412 MF_SOURCE_READER_FIRST_AUDIO_STREAM,
413 MFAudioFormat_Float,
414 codecs,
415 numCodecs);
416 if (FAILED(hr)) {
417 NS_WARNING("Failed to configure WMF Audio decoder for PCM output");
418 return hr;
419 }
421 RefPtr<IMFMediaType> mediaType;
422 hr = mSourceReader->GetCurrentMediaType(MF_SOURCE_READER_FIRST_AUDIO_STREAM,
423 byRef(mediaType));
424 if (FAILED(hr)) {
425 NS_WARNING("Failed to get configured audio media type");
426 return hr;
427 }
429 mAudioRate = MFGetAttributeUINT32(mediaType, MF_MT_AUDIO_SAMPLES_PER_SECOND, 0);
430 mAudioChannels = MFGetAttributeUINT32(mediaType, MF_MT_AUDIO_NUM_CHANNELS, 0);
431 mAudioBytesPerSample = MFGetAttributeUINT32(mediaType, MF_MT_AUDIO_BITS_PER_SAMPLE, 16) / 8;
433 mInfo.mAudio.mChannels = mAudioChannels;
434 mInfo.mAudio.mRate = mAudioRate;
435 mHasAudio = mInfo.mAudio.mHasAudio = true;
437 DECODER_LOG("Successfully configured audio stream. rate=%u channels=%u bitsPerSample=%u",
438 mAudioRate, mAudioChannels, mAudioBytesPerSample);
440 return S_OK;
441 }
443 HRESULT
444 WMFReader::CreateSourceReader()
445 {
446 HRESULT hr;
448 RefPtr<IMFAttributes> attr;
449 hr = wmf::MFCreateAttributes(byRef(attr), 1);
450 NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
452 hr = attr->SetUnknown(MF_SOURCE_READER_ASYNC_CALLBACK, mSourceReaderCallback);
453 NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
455 if (mUseHwAccel) {
456 hr = attr->SetUnknown(MF_SOURCE_READER_D3D_MANAGER,
457 mDXVA2Manager->GetDXVADeviceManager());
458 if (FAILED(hr)) {
459 DECODER_LOG("Failed to set DXVA2 D3D Device manager on source reader attributes");
460 mUseHwAccel = false;
461 }
462 }
464 hr = wmf::MFCreateSourceReaderFromByteStream(mByteStream, attr, byRef(mSourceReader));
465 NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
467 hr = ConfigureVideoDecoder();
468 NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
470 hr = ConfigureAudioDecoder();
471 NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
473 if (mUseHwAccel && mInfo.mVideo.mHasVideo) {
474 RefPtr<IMFTransform> videoDecoder;
475 hr = mSourceReader->GetServiceForStream(MF_SOURCE_READER_FIRST_VIDEO_STREAM,
476 GUID_NULL,
477 IID_IMFTransform,
478 (void**)(IMFTransform**)(byRef(videoDecoder)));
480 if (SUCCEEDED(hr)) {
481 ULONG_PTR manager = ULONG_PTR(mDXVA2Manager->GetDXVADeviceManager());
482 hr = videoDecoder->ProcessMessage(MFT_MESSAGE_SET_D3D_MANAGER,
483 manager);
484 if (hr == MF_E_TRANSFORM_TYPE_NOT_SET) {
485 // Ignore MF_E_TRANSFORM_TYPE_NOT_SET. Vista returns this here
486 // on some, perhaps all, video cards. This may be because activating
487 // DXVA changes the available output types. It seems to be safe to
488 // ignore this error.
489 hr = S_OK;
490 }
491 }
492 if (FAILED(hr)) {
493 DECODER_LOG("Failed to set DXVA2 D3D Device manager on decoder hr=0x%x", hr);
494 mUseHwAccel = false;
495 }
496 }
497 return hr;
498 }
500 nsresult
501 WMFReader::ReadMetadata(MediaInfo* aInfo,
502 MetadataTags** aTags)
503 {
504 NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
506 DECODER_LOG("WMFReader::ReadMetadata()");
507 HRESULT hr;
509 const bool triedToInitDXVA = mUseHwAccel;
510 if (FAILED(CreateSourceReader())) {
511 mSourceReader = nullptr;
512 if (triedToInitDXVA && !mUseHwAccel) {
513 // We tried to initialize DXVA and failed. Try again to create the
514 // IMFSourceReader but this time we won't use DXVA. Note that we
515 // must recreate the IMFSourceReader from scratch, as on some systems
516 // (AMD Radeon 3000) we cannot successfully reconfigure an existing
517 // reader to not use DXVA after we've failed to configure DXVA.
518 // See bug 987127.
519 if (FAILED(CreateSourceReader())) {
520 mSourceReader = nullptr;
521 }
522 }
523 }
525 if (!mSourceReader) {
526 NS_WARNING("Failed to create IMFSourceReader");
527 return NS_ERROR_FAILURE;
528 }
530 if (mInfo.HasVideo()) {
531 DECODER_LOG("Using DXVA: %s", (mUseHwAccel ? "Yes" : "No"));
532 }
534 // Abort if both video and audio failed to initialize.
535 NS_ENSURE_TRUE(mInfo.HasValidMedia(), NS_ERROR_FAILURE);
537 // Get the duration, and report it to the decoder if we have it.
538 int64_t duration = 0;
539 hr = GetSourceReaderDuration(mSourceReader, duration);
540 if (SUCCEEDED(hr)) {
541 ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
542 mDecoder->SetMediaEndTime(duration);
543 }
544 // We can seek if we get a duration *and* the reader reports that it's
545 // seekable.
546 bool canSeek = false;
547 if (FAILED(hr) ||
548 FAILED(GetSourceReaderCanSeek(mSourceReader, canSeek)) ||
549 !canSeek) {
550 mDecoder->SetMediaSeekable(false);
551 }
553 *aInfo = mInfo;
554 *aTags = nullptr;
555 // aTags can be retrieved using techniques like used here:
556 // http://blogs.msdn.com/b/mf/archive/2010/01/12/mfmediapropdump.aspx
558 return NS_OK;
559 }
561 bool
562 WMFReader::DecodeAudioData()
563 {
564 NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
566 HRESULT hr;
567 hr = mSourceReader->ReadSample(MF_SOURCE_READER_FIRST_AUDIO_STREAM,
568 0, // control flags
569 0, // read stream index
570 nullptr,
571 nullptr,
572 nullptr);
574 if (FAILED(hr)) {
575 DECODER_LOG("WMFReader::DecodeAudioData() ReadSample failed with hr=0x%x", hr);
576 // End the stream.
577 return false;
578 }
580 DWORD flags = 0;
581 LONGLONG timestampHns = 0;
582 RefPtr<IMFSample> sample;
583 hr = mSourceReaderCallback->Wait(&flags, ×tampHns, byRef(sample));
584 if (FAILED(hr) ||
585 (flags & MF_SOURCE_READERF_ERROR) ||
586 (flags & MF_SOURCE_READERF_ENDOFSTREAM) ||
587 (flags & MF_SOURCE_READERF_CURRENTMEDIATYPECHANGED)) {
588 DECODER_LOG("WMFReader::DecodeAudioData() ReadSample failed with hr=0x%x flags=0x%x",
589 hr, flags);
590 // End the stream.
591 return false;
592 }
594 if (!sample) {
595 // Not enough data? Try again...
596 return true;
597 }
599 RefPtr<IMFMediaBuffer> buffer;
600 hr = sample->ConvertToContiguousBuffer(byRef(buffer));
601 NS_ENSURE_TRUE(SUCCEEDED(hr), false);
603 BYTE* data = nullptr; // Note: *data will be owned by the IMFMediaBuffer, we don't need to free it.
604 DWORD maxLength = 0, currentLength = 0;
605 hr = buffer->Lock(&data, &maxLength, ¤tLength);
606 NS_ENSURE_TRUE(SUCCEEDED(hr), false);
608 uint32_t numFrames = currentLength / mAudioBytesPerSample / mAudioChannels;
609 NS_ASSERTION(sizeof(AudioDataValue) == mAudioBytesPerSample, "Size calculation is wrong");
610 nsAutoArrayPtr<AudioDataValue> pcmSamples(new AudioDataValue[numFrames * mAudioChannels]);
611 memcpy(pcmSamples.get(), data, currentLength);
612 buffer->Unlock();
614 // We calculate the timestamp and the duration based on the number of audio
615 // frames we've already played. We don't trust the timestamp stored on the
616 // IMFSample, as sometimes it's wrong, possibly due to buggy encoders?
618 // If this sample block comes after a discontinuity (i.e. a gap or seek)
619 // reset the frame counters, and capture the timestamp. Future timestamps
620 // will be offset from this block's timestamp.
621 UINT32 discontinuity = false;
622 sample->GetUINT32(MFSampleExtension_Discontinuity, &discontinuity);
623 if (mMustRecaptureAudioPosition || discontinuity) {
624 mAudioFrameSum = 0;
625 hr = HNsToFrames(timestampHns, mAudioRate, &mAudioFrameOffset);
626 NS_ENSURE_TRUE(SUCCEEDED(hr), false);
627 mMustRecaptureAudioPosition = false;
628 }
630 int64_t timestamp;
631 hr = FramesToUsecs(mAudioFrameOffset + mAudioFrameSum, mAudioRate, ×tamp);
632 NS_ENSURE_TRUE(SUCCEEDED(hr), false);
634 mAudioFrameSum += numFrames;
636 int64_t duration;
637 hr = FramesToUsecs(numFrames, mAudioRate, &duration);
638 NS_ENSURE_TRUE(SUCCEEDED(hr), false);
640 mAudioQueue.Push(new AudioData(mDecoder->GetResource()->Tell(),
641 timestamp,
642 duration,
643 numFrames,
644 pcmSamples.forget(),
645 mAudioChannels));
647 #ifdef LOG_SAMPLE_DECODE
648 DECODER_LOG("Decoded audio sample! timestamp=%lld duration=%lld currentLength=%u",
649 timestamp, duration, currentLength);
650 #endif
652 return true;
653 }
655 HRESULT
656 WMFReader::CreateBasicVideoFrame(IMFSample* aSample,
657 int64_t aTimestampUsecs,
658 int64_t aDurationUsecs,
659 int64_t aOffsetBytes,
660 VideoData** aOutVideoData)
661 {
662 NS_ENSURE_TRUE(aSample, E_POINTER);
663 NS_ENSURE_TRUE(aOutVideoData, E_POINTER);
665 *aOutVideoData = nullptr;
667 HRESULT hr;
668 RefPtr<IMFMediaBuffer> buffer;
670 // Must convert to contiguous buffer to use IMD2DBuffer interface.
671 hr = aSample->ConvertToContiguousBuffer(byRef(buffer));
672 NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
674 // Try and use the IMF2DBuffer interface if available, otherwise fallback
675 // to the IMFMediaBuffer interface. Apparently IMF2DBuffer is more efficient,
676 // but only some systems (Windows 8?) support it.
677 BYTE* data = nullptr;
678 LONG stride = 0;
679 RefPtr<IMF2DBuffer> twoDBuffer;
680 hr = buffer->QueryInterface(static_cast<IMF2DBuffer**>(byRef(twoDBuffer)));
681 if (SUCCEEDED(hr)) {
682 hr = twoDBuffer->Lock2D(&data, &stride);
683 NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
684 } else {
685 hr = buffer->Lock(&data, nullptr, nullptr);
686 NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
687 stride = mVideoStride;
688 }
690 // YV12, planar format: [YYYY....][VVVV....][UUUU....]
691 // i.e., Y, then V, then U.
692 VideoData::YCbCrBuffer b;
694 // Y (Y') plane
695 b.mPlanes[0].mData = data;
696 b.mPlanes[0].mStride = stride;
697 b.mPlanes[0].mHeight = mVideoHeight;
698 b.mPlanes[0].mWidth = mVideoWidth;
699 b.mPlanes[0].mOffset = 0;
700 b.mPlanes[0].mSkip = 0;
702 // The V and U planes are stored 16-row-aligned, so we need to add padding
703 // to the row heights to ensure the Y'CbCr planes are referenced properly.
704 uint32_t padding = 0;
705 if (mVideoHeight % 16 != 0) {
706 padding = 16 - (mVideoHeight % 16);
707 }
708 uint32_t y_size = stride * (mVideoHeight + padding);
709 uint32_t v_size = stride * (mVideoHeight + padding) / 4;
710 uint32_t halfStride = (stride + 1) / 2;
711 uint32_t halfHeight = (mVideoHeight + 1) / 2;
712 uint32_t halfWidth = (mVideoWidth + 1) / 2;
714 // U plane (Cb)
715 b.mPlanes[1].mData = data + y_size + v_size;
716 b.mPlanes[1].mStride = halfStride;
717 b.mPlanes[1].mHeight = halfHeight;
718 b.mPlanes[1].mWidth = halfWidth;
719 b.mPlanes[1].mOffset = 0;
720 b.mPlanes[1].mSkip = 0;
722 // V plane (Cr)
723 b.mPlanes[2].mData = data + y_size;
724 b.mPlanes[2].mStride = halfStride;
725 b.mPlanes[2].mHeight = halfHeight;
726 b.mPlanes[2].mWidth = halfWidth;
727 b.mPlanes[2].mOffset = 0;
728 b.mPlanes[2].mSkip = 0;
730 VideoData *v = VideoData::Create(mInfo.mVideo,
731 mDecoder->GetImageContainer(),
732 aOffsetBytes,
733 aTimestampUsecs,
734 aDurationUsecs,
735 b,
736 false,
737 -1,
738 ToIntRect(mPictureRegion));
739 if (twoDBuffer) {
740 twoDBuffer->Unlock2D();
741 } else {
742 buffer->Unlock();
743 }
745 *aOutVideoData = v;
747 return S_OK;
748 }
750 HRESULT
751 WMFReader::CreateD3DVideoFrame(IMFSample* aSample,
752 int64_t aTimestampUsecs,
753 int64_t aDurationUsecs,
754 int64_t aOffsetBytes,
755 VideoData** aOutVideoData)
756 {
757 NS_ENSURE_TRUE(aSample, E_POINTER);
758 NS_ENSURE_TRUE(aOutVideoData, E_POINTER);
759 NS_ENSURE_TRUE(mDXVA2Manager, E_ABORT);
760 NS_ENSURE_TRUE(mUseHwAccel, E_ABORT);
762 *aOutVideoData = nullptr;
763 HRESULT hr;
765 nsRefPtr<Image> image;
766 hr = mDXVA2Manager->CopyToImage(aSample,
767 mPictureRegion,
768 mDecoder->GetImageContainer(),
769 getter_AddRefs(image));
770 NS_ENSURE_TRUE(SUCCEEDED(hr), hr);
771 NS_ENSURE_TRUE(image, E_FAIL);
773 VideoData *v = VideoData::CreateFromImage(mInfo.mVideo,
774 mDecoder->GetImageContainer(),
775 aOffsetBytes,
776 aTimestampUsecs,
777 aDurationUsecs,
778 image.forget(),
779 false,
780 -1,
781 ToIntRect(mPictureRegion));
783 NS_ENSURE_TRUE(v, E_FAIL);
784 *aOutVideoData = v;
786 return S_OK;
787 }
789 bool
790 WMFReader::DecodeVideoFrame(bool &aKeyframeSkip,
791 int64_t aTimeThreshold)
792 {
793 NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
795 // Record number of frames decoded and parsed. Automatically update the
796 // stats counters using the AutoNotifyDecoded stack-based class.
797 uint32_t parsed = 0, decoded = 0;
798 AbstractMediaDecoder::AutoNotifyDecoded autoNotify(mDecoder, parsed, decoded);
800 HRESULT hr;
802 hr = mSourceReader->ReadSample(MF_SOURCE_READER_FIRST_VIDEO_STREAM,
803 0, // control flags
804 0, // read stream index
805 nullptr,
806 nullptr,
807 nullptr);
808 if (FAILED(hr)) {
809 DECODER_LOG("WMFReader::DecodeVideoData() ReadSample failed with hr=0x%x", hr);
810 return false;
811 }
813 DWORD flags = 0;
814 LONGLONG timestampHns = 0;
815 RefPtr<IMFSample> sample;
816 hr = mSourceReaderCallback->Wait(&flags, ×tampHns, byRef(sample));
818 if (flags & MF_SOURCE_READERF_ERROR) {
819 NS_WARNING("WMFReader: Catastrophic failure reading video sample");
820 // Future ReadSample() calls will fail, so give up and report end of stream.
821 return false;
822 }
824 if (FAILED(hr)) {
825 // Unknown failure, ask caller to try again?
826 return true;
827 }
829 if (!sample) {
830 if ((flags & MF_SOURCE_READERF_ENDOFSTREAM)) {
831 DECODER_LOG("WMFReader; Null sample after video decode, at end of stream");
832 return false;
833 }
834 DECODER_LOG("WMFReader; Null sample after video decode. Maybe insufficient data...");
835 return true;
836 }
838 if ((flags & MF_SOURCE_READERF_CURRENTMEDIATYPECHANGED)) {
839 DECODER_LOG("WMFReader: Video media type changed!");
840 RefPtr<IMFMediaType> mediaType;
841 hr = mSourceReader->GetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM,
842 byRef(mediaType));
843 if (FAILED(hr) ||
844 FAILED(ConfigureVideoFrameGeometry(mediaType))) {
845 NS_WARNING("Failed to reconfigure video media type");
846 return false;
847 }
848 }
850 int64_t timestamp = HNsToUsecs(timestampHns);
851 if (timestamp < aTimeThreshold) {
852 return true;
853 }
854 int64_t offset = mDecoder->GetResource()->Tell();
855 int64_t duration = GetSampleDuration(sample);
857 VideoData* v = nullptr;
858 if (mUseHwAccel) {
859 hr = CreateD3DVideoFrame(sample, timestamp, duration, offset, &v);
860 } else {
861 hr = CreateBasicVideoFrame(sample, timestamp, duration, offset, &v);
862 }
863 NS_ENSURE_TRUE(SUCCEEDED(hr) && v, false);
865 parsed++;
866 decoded++;
867 mVideoQueue.Push(v);
869 #ifdef LOG_SAMPLE_DECODE
870 DECODER_LOG("Decoded video sample timestamp=%lld duration=%lld stride=%d height=%u flags=%u",
871 timestamp, duration, mVideoStride, mVideoHeight, flags);
872 #endif
874 if ((flags & MF_SOURCE_READERF_ENDOFSTREAM)) {
875 // End of stream.
876 DECODER_LOG("End of video stream");
877 return false;
878 }
880 return true;
881 }
883 nsresult
884 WMFReader::Seek(int64_t aTargetUs,
885 int64_t aStartTime,
886 int64_t aEndTime,
887 int64_t aCurrentTime)
888 {
889 DECODER_LOG("WMFReader::Seek() %lld", aTargetUs);
891 NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
892 #ifdef DEBUG
893 bool canSeek = false;
894 GetSourceReaderCanSeek(mSourceReader, canSeek);
895 NS_ASSERTION(canSeek, "WMFReader::Seek() should only be called if we can seek!");
896 #endif
898 nsresult rv = ResetDecode();
899 NS_ENSURE_SUCCESS(rv, rv);
901 // Mark that we must recapture the audio frame count from the next sample.
902 // WMF doesn't set a discontinuity marker when we seek to time 0, so we
903 // must remember to recapture the audio frame offset and reset the frame
904 // sum on the next audio packet we decode.
905 mMustRecaptureAudioPosition = true;
907 AutoPropVar var;
908 HRESULT hr = InitPropVariantFromInt64(UsecsToHNs(aTargetUs), &var);
909 NS_ENSURE_TRUE(SUCCEEDED(hr), NS_ERROR_FAILURE);
911 hr = mSourceReader->SetCurrentPosition(GUID_NULL, var);
912 NS_ENSURE_TRUE(SUCCEEDED(hr), NS_ERROR_FAILURE);
914 return NS_OK;
915 }
917 } // namespace mozilla
