content/media/webm/WebMReader.cpp@44a2da4a2ab2
content/media/webm/WebMReader.cpp
Fri, 16 Jan 2015 04:50:19 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Fri, 16 Jan 2015 04:50:19 +0100
- branch
- TOR_BUG_9701
- changeset 13
- 44a2da4a2ab2
- permissions
- -rw-r--r--
Replace accessor implementation with direct member state manipulation, by
request https://trac.torproject.org/projects/tor/ticket/9701#comment:32
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #include "nsError.h"
7 #include "MediaDecoderStateMachine.h"
8 #include "AbstractMediaDecoder.h"
9 #include "MediaResource.h"
10 #include "WebMReader.h"
11 #include "WebMBufferedParser.h"
12 #include "mozilla/dom/TimeRanges.h"
13 #include "VorbisUtils.h"
14 #include "gfx2DGlue.h"
16 #include <algorithm>
18 #define VPX_DONT_DEFINE_STDINT_TYPES
19 #include "vpx/vp8dx.h"
20 #include "vpx/vpx_decoder.h"
22 #include "OggReader.h"
24 using mozilla::NesteggPacketHolder;
26 template <>
27 class nsAutoRefTraits<NesteggPacketHolder> : public nsPointerRefTraits<NesteggPacketHolder>
28 {
29 public:
30 static void Release(NesteggPacketHolder* aHolder) { delete aHolder; }
31 };
33 namespace mozilla {
35 using namespace gfx;
36 using namespace layers;
38 // Un-comment to enable logging of seek bisections.
39 //#define SEEK_LOGGING
41 #ifdef PR_LOGGING
42 extern PRLogModuleInfo* gMediaDecoderLog;
43 PRLogModuleInfo* gNesteggLog;
44 #define LOG(type, msg) PR_LOG(gMediaDecoderLog, type, msg)
45 #ifdef SEEK_LOGGING
46 #define SEEK_LOG(type, msg) PR_LOG(gMediaDecoderLog, type, msg)
47 #else
48 #define SEEK_LOG(type, msg)
49 #endif
50 #else
51 #define LOG(type, msg)
52 #define SEEK_LOG(type, msg)
53 #endif
55 static const unsigned NS_PER_USEC = 1000;
56 static const double NS_PER_S = 1e9;
58 // Functions for reading and seeking using MediaResource required for
59 // nestegg_io. The 'user data' passed to these functions is the
60 // decoder from which the media resource is obtained.
61 static int webm_read(void *aBuffer, size_t aLength, void *aUserData)
62 {
63 NS_ASSERTION(aUserData, "aUserData must point to a valid AbstractMediaDecoder");
64 AbstractMediaDecoder* decoder = reinterpret_cast<AbstractMediaDecoder*>(aUserData);
65 MediaResource* resource = decoder->GetResource();
66 NS_ASSERTION(resource, "Decoder has no media resource");
68 nsresult rv = NS_OK;
69 bool eof = false;
71 char *p = static_cast<char *>(aBuffer);
72 while (NS_SUCCEEDED(rv) && aLength > 0) {
73 uint32_t bytes = 0;
74 rv = resource->Read(p, aLength, &bytes);
75 if (bytes == 0) {
76 eof = true;
77 break;
78 }
79 aLength -= bytes;
80 p += bytes;
81 }
83 return NS_FAILED(rv) ? -1 : eof ? 0 : 1;
84 }
86 static int webm_seek(int64_t aOffset, int aWhence, void *aUserData)
87 {
88 NS_ASSERTION(aUserData, "aUserData must point to a valid AbstractMediaDecoder");
89 AbstractMediaDecoder* decoder = reinterpret_cast<AbstractMediaDecoder*>(aUserData);
90 MediaResource* resource = decoder->GetResource();
91 NS_ASSERTION(resource, "Decoder has no media resource");
92 nsresult rv = resource->Seek(aWhence, aOffset);
93 return NS_SUCCEEDED(rv) ? 0 : -1;
94 }
96 static int64_t webm_tell(void *aUserData)
97 {
98 NS_ASSERTION(aUserData, "aUserData must point to a valid AbstractMediaDecoder");
99 AbstractMediaDecoder* decoder = reinterpret_cast<AbstractMediaDecoder*>(aUserData);
100 MediaResource* resource = decoder->GetResource();
101 NS_ASSERTION(resource, "Decoder has no media resource");
102 return resource->Tell();
103 }
105 static void webm_log(nestegg * context,
106 unsigned int severity,
107 char const * format, ...)
108 {
109 #ifdef PR_LOGGING
110 va_list args;
111 char msg[256];
112 const char * sevStr;
114 switch(severity) {
115 case NESTEGG_LOG_DEBUG:
116 sevStr = "DBG";
117 break;
118 case NESTEGG_LOG_INFO:
119 sevStr = "INF";
120 break;
121 case NESTEGG_LOG_WARNING:
122 sevStr = "WRN";
123 break;
124 case NESTEGG_LOG_ERROR:
125 sevStr = "ERR";
126 break;
127 case NESTEGG_LOG_CRITICAL:
128 sevStr = "CRT";
129 break;
130 default:
131 sevStr = "UNK";
132 break;
133 }
135 va_start(args, format);
137 PR_snprintf(msg, sizeof(msg), "%p [Nestegg-%s] ", context, sevStr);
138 PR_vsnprintf(msg+strlen(msg), sizeof(msg)-strlen(msg), format, args);
139 PR_LOG(gNesteggLog, PR_LOG_DEBUG, (msg));
141 va_end(args);
142 #endif
143 }
145 WebMReader::WebMReader(AbstractMediaDecoder* aDecoder)
146 : MediaDecoderReader(aDecoder),
147 mContext(nullptr),
148 mPacketCount(0),
149 mChannels(0),
150 #ifdef MOZ_OPUS
151 mOpusParser(nullptr),
152 mOpusDecoder(nullptr),
153 mSkip(0),
154 mSeekPreroll(0),
155 #endif
156 mVideoTrack(0),
157 mAudioTrack(0),
158 mAudioStartUsec(-1),
159 mAudioFrames(0),
160 mAudioCodec(-1),
161 mVideoCodec(-1),
162 mHasVideo(false),
163 mHasAudio(false)
164 {
165 MOZ_COUNT_CTOR(WebMReader);
166 #ifdef PR_LOGGING
167 if (!gNesteggLog) {
168 gNesteggLog = PR_NewLogModule("Nestegg");
169 }
170 #endif
171 // Zero these member vars to avoid crashes in VP8 destroy and Vorbis clear
172 // functions when destructor is called before |Init|.
173 memset(&mVPX, 0, sizeof(vpx_codec_ctx_t));
174 memset(&mVorbisBlock, 0, sizeof(vorbis_block));
175 memset(&mVorbisDsp, 0, sizeof(vorbis_dsp_state));
176 memset(&mVorbisInfo, 0, sizeof(vorbis_info));
177 memset(&mVorbisComment, 0, sizeof(vorbis_comment));
178 }
180 WebMReader::~WebMReader()
181 {
182 Cleanup();
184 mVideoPackets.Reset();
185 mAudioPackets.Reset();
187 vpx_codec_destroy(&mVPX);
189 vorbis_block_clear(&mVorbisBlock);
190 vorbis_dsp_clear(&mVorbisDsp);
191 vorbis_info_clear(&mVorbisInfo);
192 vorbis_comment_clear(&mVorbisComment);
194 if (mOpusDecoder) {
195 opus_multistream_decoder_destroy(mOpusDecoder);
196 mOpusDecoder = nullptr;
197 }
199 MOZ_COUNT_DTOR(WebMReader);
200 }
202 nsresult WebMReader::Init(MediaDecoderReader* aCloneDonor)
203 {
205 vorbis_info_init(&mVorbisInfo);
206 vorbis_comment_init(&mVorbisComment);
207 memset(&mVorbisDsp, 0, sizeof(vorbis_dsp_state));
208 memset(&mVorbisBlock, 0, sizeof(vorbis_block));
210 if (aCloneDonor) {
211 mBufferedState = static_cast<WebMReader*>(aCloneDonor)->mBufferedState;
212 } else {
213 mBufferedState = new WebMBufferedState;
214 }
216 return NS_OK;
217 }
219 nsresult WebMReader::ResetDecode()
220 {
221 mAudioFrames = 0;
222 mAudioStartUsec = -1;
223 nsresult res = NS_OK;
224 if (NS_FAILED(MediaDecoderReader::ResetDecode())) {
225 res = NS_ERROR_FAILURE;
226 }
228 if (mAudioCodec == NESTEGG_CODEC_VORBIS) {
229 // Ignore failed results from vorbis_synthesis_restart. They
230 // aren't fatal and it fails when ResetDecode is called at a
231 // time when no vorbis data has been read.
232 vorbis_synthesis_restart(&mVorbisDsp);
233 #ifdef MOZ_OPUS
234 } else if (mAudioCodec == NESTEGG_CODEC_OPUS) {
235 if (mOpusDecoder) {
236 // Reset the decoder.
237 opus_multistream_decoder_ctl(mOpusDecoder, OPUS_RESET_STATE);
238 mSkip = mOpusParser->mPreSkip;
239 }
240 #endif
241 }
243 mVideoPackets.Reset();
244 mAudioPackets.Reset();
246 return res;
247 }
249 void WebMReader::Cleanup()
250 {
251 if (mContext) {
252 nestegg_destroy(mContext);
253 mContext = nullptr;
254 }
255 }
257 nsresult WebMReader::ReadMetadata(MediaInfo* aInfo,
258 MetadataTags** aTags)
259 {
260 NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
262 nestegg_io io;
263 io.read = webm_read;
264 io.seek = webm_seek;
265 io.tell = webm_tell;
266 io.userdata = mDecoder;
267 int64_t maxOffset = -1;
268 int r = nestegg_init(&mContext, io, &webm_log, maxOffset);
269 if (r == -1) {
270 return NS_ERROR_FAILURE;
271 }
273 uint64_t duration = 0;
274 r = nestegg_duration(mContext, &duration);
275 if (r == 0) {
276 ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
277 mDecoder->SetMediaDuration(duration / NS_PER_USEC);
278 }
280 unsigned int ntracks = 0;
281 r = nestegg_track_count(mContext, &ntracks);
282 if (r == -1) {
283 Cleanup();
284 return NS_ERROR_FAILURE;
285 }
287 for (uint32_t track = 0; track < ntracks; ++track) {
288 int id = nestegg_track_codec_id(mContext, track);
289 if (id == -1) {
290 Cleanup();
291 return NS_ERROR_FAILURE;
292 }
293 int type = nestegg_track_type(mContext, track);
294 if (!mHasVideo && type == NESTEGG_TRACK_VIDEO) {
295 nestegg_video_params params;
296 r = nestegg_track_video_params(mContext, track, ¶ms);
297 if (r == -1) {
298 Cleanup();
299 return NS_ERROR_FAILURE;
300 }
302 vpx_codec_iface_t* dx = nullptr;
303 mVideoCodec = nestegg_track_codec_id(mContext, track);
304 if (mVideoCodec == NESTEGG_CODEC_VP8) {
305 dx = vpx_codec_vp8_dx();
306 } else if (mVideoCodec == NESTEGG_CODEC_VP9) {
307 dx = vpx_codec_vp9_dx();
308 }
309 if (!dx || vpx_codec_dec_init(&mVPX, dx, nullptr, 0)) {
310 Cleanup();
311 return NS_ERROR_FAILURE;
312 }
314 // Picture region, taking into account cropping, before scaling
315 // to the display size.
316 nsIntRect pictureRect(params.crop_left,
317 params.crop_top,
318 params.width - (params.crop_right + params.crop_left),
319 params.height - (params.crop_bottom + params.crop_top));
321 // If the cropping data appears invalid then use the frame data
322 if (pictureRect.width <= 0 ||
323 pictureRect.height <= 0 ||
324 pictureRect.x < 0 ||
325 pictureRect.y < 0)
326 {
327 pictureRect.x = 0;
328 pictureRect.y = 0;
329 pictureRect.width = params.width;
330 pictureRect.height = params.height;
331 }
333 // Validate the container-reported frame and pictureRect sizes. This ensures
334 // that our video frame creation code doesn't overflow.
335 nsIntSize displaySize(params.display_width, params.display_height);
336 nsIntSize frameSize(params.width, params.height);
337 if (!IsValidVideoRegion(frameSize, pictureRect, displaySize)) {
338 // Video track's frame sizes will overflow. Ignore the video track.
339 continue;
340 }
342 mVideoTrack = track;
343 mHasVideo = true;
344 mInfo.mVideo.mHasVideo = true;
346 mInfo.mVideo.mDisplay = displaySize;
347 mPicture = pictureRect;
348 mInitialFrame = frameSize;
350 switch (params.stereo_mode) {
351 case NESTEGG_VIDEO_MONO:
352 mInfo.mVideo.mStereoMode = StereoMode::MONO;
353 break;
354 case NESTEGG_VIDEO_STEREO_LEFT_RIGHT:
355 mInfo.mVideo.mStereoMode = StereoMode::LEFT_RIGHT;
356 break;
357 case NESTEGG_VIDEO_STEREO_BOTTOM_TOP:
358 mInfo.mVideo.mStereoMode = StereoMode::BOTTOM_TOP;
359 break;
360 case NESTEGG_VIDEO_STEREO_TOP_BOTTOM:
361 mInfo.mVideo.mStereoMode = StereoMode::TOP_BOTTOM;
362 break;
363 case NESTEGG_VIDEO_STEREO_RIGHT_LEFT:
364 mInfo.mVideo.mStereoMode = StereoMode::RIGHT_LEFT;
365 break;
366 }
367 }
368 else if (!mHasAudio && type == NESTEGG_TRACK_AUDIO) {
369 nestegg_audio_params params;
370 r = nestegg_track_audio_params(mContext, track, ¶ms);
371 if (r == -1) {
372 Cleanup();
373 return NS_ERROR_FAILURE;
374 }
376 mAudioTrack = track;
377 mHasAudio = true;
378 mInfo.mAudio.mHasAudio = true;
379 mAudioCodec = nestegg_track_codec_id(mContext, track);
380 mCodecDelay = params.codec_delay / NS_PER_USEC;
382 if (mAudioCodec == NESTEGG_CODEC_VORBIS) {
383 // Get the Vorbis header data
384 unsigned int nheaders = 0;
385 r = nestegg_track_codec_data_count(mContext, track, &nheaders);
386 if (r == -1 || nheaders != 3) {
387 Cleanup();
388 return NS_ERROR_FAILURE;
389 }
391 for (uint32_t header = 0; header < nheaders; ++header) {
392 unsigned char* data = 0;
393 size_t length = 0;
395 r = nestegg_track_codec_data(mContext, track, header, &data, &length);
396 if (r == -1) {
397 Cleanup();
398 return NS_ERROR_FAILURE;
399 }
400 ogg_packet opacket = InitOggPacket(data, length, header == 0, false, 0);
402 r = vorbis_synthesis_headerin(&mVorbisInfo,
403 &mVorbisComment,
404 &opacket);
405 if (r != 0) {
406 Cleanup();
407 return NS_ERROR_FAILURE;
408 }
409 }
411 r = vorbis_synthesis_init(&mVorbisDsp, &mVorbisInfo);
412 if (r != 0) {
413 Cleanup();
414 return NS_ERROR_FAILURE;
415 }
417 r = vorbis_block_init(&mVorbisDsp, &mVorbisBlock);
418 if (r != 0) {
419 Cleanup();
420 return NS_ERROR_FAILURE;
421 }
423 mInfo.mAudio.mRate = mVorbisDsp.vi->rate;
424 mInfo.mAudio.mChannels = mVorbisDsp.vi->channels;
425 mChannels = mInfo.mAudio.mChannels;
426 #ifdef MOZ_OPUS
427 } else if (mAudioCodec == NESTEGG_CODEC_OPUS) {
428 unsigned char* data = 0;
429 size_t length = 0;
430 r = nestegg_track_codec_data(mContext, track, 0, &data, &length);
431 if (r == -1) {
432 Cleanup();
433 return NS_ERROR_FAILURE;
434 }
436 mOpusParser = new OpusParser;
437 if (!mOpusParser->DecodeHeader(data, length)) {
438 Cleanup();
439 return NS_ERROR_FAILURE;
440 }
442 if (!InitOpusDecoder()) {
443 Cleanup();
444 return NS_ERROR_FAILURE;
445 }
447 if (static_cast<int64_t>(mCodecDelay) != FramesToUsecs(mOpusParser->mPreSkip, mOpusParser->mRate).value()) {
448 LOG(PR_LOG_WARNING,
449 ("Invalid Opus header: CodecDelay and pre-skip do not match!\n"));
450 Cleanup();
451 return NS_ERROR_FAILURE;
452 }
454 mInfo.mAudio.mRate = mOpusParser->mRate;
456 mInfo.mAudio.mChannels = mOpusParser->mChannels;
457 mChannels = mInfo.mAudio.mChannels;
458 mSeekPreroll = params.seek_preroll;
459 #endif
460 } else {
461 Cleanup();
462 return NS_ERROR_FAILURE;
463 }
464 }
465 }
467 // We can't seek in buffered regions if we have no cues.
468 mDecoder->SetMediaSeekable(nestegg_has_cues(mContext) == 1);
470 *aInfo = mInfo;
472 *aTags = nullptr;
474 return NS_OK;
475 }
477 #ifdef MOZ_OPUS
478 bool WebMReader::InitOpusDecoder()
479 {
480 int r;
482 NS_ASSERTION(mOpusDecoder == nullptr, "leaking OpusDecoder");
484 mOpusDecoder = opus_multistream_decoder_create(mOpusParser->mRate,
485 mOpusParser->mChannels,
486 mOpusParser->mStreams,
487 mOpusParser->mCoupledStreams,
488 mOpusParser->mMappingTable,
489 &r);
490 mSkip = mOpusParser->mPreSkip;
492 return r == OPUS_OK;
493 }
494 #endif
496 ogg_packet WebMReader::InitOggPacket(unsigned char* aData,
497 size_t aLength,
498 bool aBOS,
499 bool aEOS,
500 int64_t aGranulepos)
501 {
502 ogg_packet packet;
503 packet.packet = aData;
504 packet.bytes = aLength;
505 packet.b_o_s = aBOS;
506 packet.e_o_s = aEOS;
507 packet.granulepos = aGranulepos;
508 packet.packetno = mPacketCount++;
509 return packet;
510 }
512 bool WebMReader::DecodeAudioPacket(nestegg_packet* aPacket, int64_t aOffset)
513 {
514 NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
516 int r = 0;
517 unsigned int count = 0;
518 r = nestegg_packet_count(aPacket, &count);
519 if (r == -1) {
520 return false;
521 }
523 uint64_t tstamp = 0;
524 r = nestegg_packet_tstamp(aPacket, &tstamp);
525 if (r == -1) {
526 return false;
527 }
529 const uint32_t rate = mInfo.mAudio.mRate;
530 uint64_t tstamp_usecs = tstamp / NS_PER_USEC;
531 if (mAudioStartUsec == -1) {
532 // This is the first audio chunk. Assume the start time of our decode
533 // is the start of this chunk.
534 mAudioStartUsec = tstamp_usecs;
535 }
536 // If there's a gap between the start of this audio chunk and the end of
537 // the previous audio chunk, we need to increment the packet count so that
538 // the vorbis decode doesn't use data from before the gap to help decode
539 // from after the gap.
540 CheckedInt64 tstamp_frames = UsecsToFrames(tstamp_usecs, rate);
541 CheckedInt64 decoded_frames = UsecsToFrames(mAudioStartUsec, rate);
542 if (!tstamp_frames.isValid() || !decoded_frames.isValid()) {
543 NS_WARNING("Int overflow converting WebM times to frames");
544 return false;
545 }
546 decoded_frames += mAudioFrames;
547 if (!decoded_frames.isValid()) {
548 NS_WARNING("Int overflow adding decoded_frames");
549 return false;
550 }
551 if (tstamp_frames.value() > decoded_frames.value()) {
552 #ifdef DEBUG
553 CheckedInt64 usecs = FramesToUsecs(tstamp_frames.value() - decoded_frames.value(), rate);
554 LOG(PR_LOG_DEBUG, ("WebMReader detected gap of %lld, %lld frames, in audio stream\n",
555 usecs.isValid() ? usecs.value() : -1,
556 tstamp_frames.value() - decoded_frames.value()));
557 #endif
558 mPacketCount++;
559 mAudioStartUsec = tstamp_usecs;
560 mAudioFrames = 0;
561 }
563 int32_t total_frames = 0;
564 for (uint32_t i = 0; i < count; ++i) {
565 unsigned char* data;
566 size_t length;
567 r = nestegg_packet_data(aPacket, i, &data, &length);
568 if (r == -1) {
569 return false;
570 }
571 if (mAudioCodec == NESTEGG_CODEC_VORBIS) {
572 ogg_packet opacket = InitOggPacket(data, length, false, false, -1);
574 if (vorbis_synthesis(&mVorbisBlock, &opacket) != 0) {
575 return false;
576 }
578 if (vorbis_synthesis_blockin(&mVorbisDsp,
579 &mVorbisBlock) != 0) {
580 return false;
581 }
583 VorbisPCMValue** pcm = 0;
584 int32_t frames = 0;
585 while ((frames = vorbis_synthesis_pcmout(&mVorbisDsp, &pcm)) > 0) {
586 nsAutoArrayPtr<AudioDataValue> buffer(new AudioDataValue[frames * mChannels]);
587 for (uint32_t j = 0; j < mChannels; ++j) {
588 VorbisPCMValue* channel = pcm[j];
589 for (uint32_t i = 0; i < uint32_t(frames); ++i) {
590 buffer[i*mChannels + j] = MOZ_CONVERT_VORBIS_SAMPLE(channel[i]);
591 }
592 }
594 CheckedInt64 duration = FramesToUsecs(frames, rate);
595 if (!duration.isValid()) {
596 NS_WARNING("Int overflow converting WebM audio duration");
597 return false;
598 }
599 CheckedInt64 total_duration = FramesToUsecs(total_frames, rate);
600 if (!total_duration.isValid()) {
601 NS_WARNING("Int overflow converting WebM audio total_duration");
602 return false;
603 }
605 CheckedInt64 time = total_duration + tstamp_usecs;
606 if (!time.isValid()) {
607 NS_WARNING("Int overflow adding total_duration and tstamp_usecs");
608 return false;
609 };
611 total_frames += frames;
612 AudioQueue().Push(new AudioData(aOffset,
613 time.value(),
614 duration.value(),
615 frames,
616 buffer.forget(),
617 mChannels));
618 mAudioFrames += frames;
619 if (vorbis_synthesis_read(&mVorbisDsp, frames) != 0) {
620 return false;
621 }
622 }
623 } else if (mAudioCodec == NESTEGG_CODEC_OPUS) {
624 #ifdef MOZ_OPUS
625 uint32_t channels = mOpusParser->mChannels;
627 // Maximum value is 63*2880, so there's no chance of overflow.
628 int32_t frames_number = opus_packet_get_nb_frames(data, length);
630 if (frames_number <= 0)
631 return false; // Invalid packet header.
632 int32_t samples = opus_packet_get_samples_per_frame(data,
633 (opus_int32) rate);
634 int32_t frames = frames_number*samples;
636 // A valid Opus packet must be between 2.5 and 120 ms long.
637 if (frames < 120 || frames > 5760)
638 return false;
639 nsAutoArrayPtr<AudioDataValue> buffer(new AudioDataValue[frames * channels]);
641 // Decode to the appropriate sample type.
642 #ifdef MOZ_SAMPLE_TYPE_FLOAT32
643 int ret = opus_multistream_decode_float(mOpusDecoder,
644 data, length,
645 buffer, frames, false);
646 #else
647 int ret = opus_multistream_decode(mOpusDecoder,
648 data, length,
649 buffer, frames, false);
650 #endif
651 if (ret < 0)
652 return false;
653 NS_ASSERTION(ret == frames, "Opus decoded too few audio samples");
654 CheckedInt64 startTime = tstamp_usecs;
656 // Trim the initial frames while the decoder is settling.
657 if (mSkip > 0) {
658 int32_t skipFrames = std::min(mSkip, frames);
659 if (skipFrames == frames) {
660 // discard the whole packet
661 mSkip -= frames;
662 LOG(PR_LOG_DEBUG, ("Opus decoder skipping %d frames"
663 " (whole packet)", frames));
664 return true;
665 }
666 int32_t keepFrames = frames - skipFrames;
667 if (keepFrames < 0) {
668 NS_WARNING("Int overflow in keepFrames");
669 return false;
670 }
671 int samples = keepFrames * channels;
672 if (samples < 0) {
673 NS_WARNING("Int overflow in samples");
674 return false;
675 }
676 nsAutoArrayPtr<AudioDataValue> trimBuffer(new AudioDataValue[samples]);
677 for (int i = 0; i < samples; i++)
678 trimBuffer[i] = buffer[skipFrames*channels + i];
679 startTime = startTime + FramesToUsecs(skipFrames, rate);
680 frames = keepFrames;
681 buffer = trimBuffer;
683 mSkip -= skipFrames;
684 LOG(PR_LOG_DEBUG, ("Opus decoder skipping %d frames", skipFrames));
685 }
687 int64_t discardPadding = 0;
688 r = nestegg_packet_discard_padding(aPacket, &discardPadding);
689 if (discardPadding > 0) {
690 CheckedInt64 discardFrames = UsecsToFrames(discardPadding * NS_PER_USEC, rate);
691 if (!discardFrames.isValid()) {
692 NS_WARNING("Int overflow in DiscardPadding");
693 return false;
694 }
695 int32_t keepFrames = frames - discardFrames.value();
696 if (keepFrames > 0) {
697 int samples = keepFrames * channels;
698 if (samples < 0) {
699 NS_WARNING("Int overflow in samples");
700 return false;
701 }
702 nsAutoArrayPtr<AudioDataValue> trimBuffer(new AudioDataValue[samples]);
703 for (int i = 0; i < samples; i++)
704 trimBuffer[i] = buffer[i];
705 frames = keepFrames;
706 buffer = trimBuffer;
707 } else {
708 LOG(PR_LOG_DEBUG, ("Opus decoder discarding whole packet"
709 " ( %d frames) as padding", frames));
710 return true;
711 }
712 }
714 // Apply the header gain if one was specified.
715 #ifdef MOZ_SAMPLE_TYPE_FLOAT32
716 if (mOpusParser->mGain != 1.0f) {
717 float gain = mOpusParser->mGain;
718 int samples = frames * channels;
719 for (int i = 0; i < samples; i++) {
720 buffer[i] *= gain;
721 }
722 }
723 #else
724 if (mOpusParser->mGain_Q16 != 65536) {
725 int64_t gain_Q16 = mOpusParser->mGain_Q16;
726 int samples = frames * channels;
727 for (int i = 0; i < samples; i++) {
728 int32_t val = static_cast<int32_t>((gain_Q16*buffer[i] + 32768)>>16);
729 buffer[i] = static_cast<AudioDataValue>(MOZ_CLIP_TO_15(val));
730 }
731 }
732 #endif
734 // No channel mapping for more than 8 channels.
735 if (channels > 8) {
736 return false;
737 }
739 CheckedInt64 duration = FramesToUsecs(frames, rate);
740 if (!duration.isValid()) {
741 NS_WARNING("Int overflow converting WebM audio duration");
742 return false;
743 }
744 CheckedInt64 time = startTime - mCodecDelay;
745 if (!time.isValid()) {
746 NS_WARNING("Int overflow shifting tstamp by codec delay");
747 return false;
748 };
749 AudioQueue().Push(new AudioData(mDecoder->GetResource()->Tell(),
750 time.value(),
751 duration.value(),
752 frames,
753 buffer.forget(),
754 mChannels));
756 mAudioFrames += frames;
757 #else
758 return false;
759 #endif /* MOZ_OPUS */
760 }
761 }
763 return true;
764 }
766 nsReturnRef<NesteggPacketHolder> WebMReader::NextPacket(TrackType aTrackType)
767 {
768 // The packet queue that packets will be pushed on if they
769 // are not the type we are interested in.
770 WebMPacketQueue& otherPackets =
771 aTrackType == VIDEO ? mAudioPackets : mVideoPackets;
773 // The packet queue for the type that we are interested in.
774 WebMPacketQueue &packets =
775 aTrackType == VIDEO ? mVideoPackets : mAudioPackets;
777 // Flag to indicate that we do need to playback these types of
778 // packets.
779 bool hasType = aTrackType == VIDEO ? mHasVideo : mHasAudio;
781 // Flag to indicate that we do need to playback the other type
782 // of track.
783 bool hasOtherType = aTrackType == VIDEO ? mHasAudio : mHasVideo;
785 // Track we are interested in
786 uint32_t ourTrack = aTrackType == VIDEO ? mVideoTrack : mAudioTrack;
788 // Value of other track
789 uint32_t otherTrack = aTrackType == VIDEO ? mAudioTrack : mVideoTrack;
791 nsAutoRef<NesteggPacketHolder> holder;
793 if (packets.GetSize() > 0) {
794 holder.own(packets.PopFront());
795 } else {
796 // Keep reading packets until we find a packet
797 // for the track we want.
798 do {
799 nestegg_packet* packet;
800 int r = nestegg_read_packet(mContext, &packet);
801 if (r <= 0) {
802 return nsReturnRef<NesteggPacketHolder>();
803 }
804 int64_t offset = mDecoder->GetResource()->Tell();
805 holder.own(new NesteggPacketHolder(packet, offset));
807 unsigned int track = 0;
808 r = nestegg_packet_track(packet, &track);
809 if (r == -1) {
810 return nsReturnRef<NesteggPacketHolder>();
811 }
813 if (hasOtherType && otherTrack == track) {
814 // Save the packet for when we want these packets
815 otherPackets.Push(holder.disown());
816 continue;
817 }
819 // The packet is for the track we want to play
820 if (hasType && ourTrack == track) {
821 break;
822 }
823 } while (true);
824 }
826 return holder.out();
827 }
829 bool WebMReader::DecodeAudioData()
830 {
831 NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
833 nsAutoRef<NesteggPacketHolder> holder(NextPacket(AUDIO));
834 if (!holder) {
835 return false;
836 }
838 return DecodeAudioPacket(holder->mPacket, holder->mOffset);
839 }
841 bool WebMReader::DecodeVideoFrame(bool &aKeyframeSkip,
842 int64_t aTimeThreshold)
843 {
844 NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
846 // Record number of frames decoded and parsed. Automatically update the
847 // stats counters using the AutoNotifyDecoded stack-based class.
848 uint32_t parsed = 0, decoded = 0;
849 AbstractMediaDecoder::AutoNotifyDecoded autoNotify(mDecoder, parsed, decoded);
851 nsAutoRef<NesteggPacketHolder> holder(NextPacket(VIDEO));
852 if (!holder) {
853 return false;
854 }
856 nestegg_packet* packet = holder->mPacket;
857 unsigned int track = 0;
858 int r = nestegg_packet_track(packet, &track);
859 if (r == -1) {
860 return false;
861 }
863 unsigned int count = 0;
864 r = nestegg_packet_count(packet, &count);
865 if (r == -1) {
866 return false;
867 }
869 uint64_t tstamp = 0;
870 r = nestegg_packet_tstamp(packet, &tstamp);
871 if (r == -1) {
872 return false;
873 }
875 // The end time of this frame is the start time of the next frame. Fetch
876 // the timestamp of the next packet for this track. If we've reached the
877 // end of the resource, use the file's duration as the end time of this
878 // video frame.
879 uint64_t next_tstamp = 0;
880 nsAutoRef<NesteggPacketHolder> next_holder(NextPacket(VIDEO));
881 if (next_holder) {
882 r = nestegg_packet_tstamp(next_holder->mPacket, &next_tstamp);
883 if (r == -1) {
884 return false;
885 }
886 PushVideoPacket(next_holder.disown());
887 } else {
888 ReentrantMonitorAutoEnter decoderMon(mDecoder->GetReentrantMonitor());
889 int64_t endTime = mDecoder->GetEndMediaTime();
890 if (endTime == -1) {
891 return false;
892 }
893 next_tstamp = endTime * NS_PER_USEC;
894 }
896 int64_t tstamp_usecs = tstamp / NS_PER_USEC;
897 for (uint32_t i = 0; i < count; ++i) {
898 unsigned char* data;
899 size_t length;
900 r = nestegg_packet_data(packet, i, &data, &length);
901 if (r == -1) {
902 return false;
903 }
905 vpx_codec_stream_info_t si;
906 memset(&si, 0, sizeof(si));
907 si.sz = sizeof(si);
908 if (mVideoCodec == NESTEGG_CODEC_VP8) {
909 vpx_codec_peek_stream_info(vpx_codec_vp8_dx(), data, length, &si);
910 } else if (mVideoCodec == NESTEGG_CODEC_VP9) {
911 vpx_codec_peek_stream_info(vpx_codec_vp9_dx(), data, length, &si);
912 }
913 if (aKeyframeSkip && (!si.is_kf || tstamp_usecs < aTimeThreshold)) {
914 // Skipping to next keyframe...
915 parsed++; // Assume 1 frame per chunk.
916 continue;
917 }
919 if (aKeyframeSkip && si.is_kf) {
920 aKeyframeSkip = false;
921 }
923 if (vpx_codec_decode(&mVPX, data, length, nullptr, 0)) {
924 return false;
925 }
927 // If the timestamp of the video frame is less than
928 // the time threshold required then it is not added
929 // to the video queue and won't be displayed.
930 if (tstamp_usecs < aTimeThreshold) {
931 parsed++; // Assume 1 frame per chunk.
932 continue;
933 }
935 vpx_codec_iter_t iter = nullptr;
936 vpx_image_t *img;
938 while ((img = vpx_codec_get_frame(&mVPX, &iter))) {
939 NS_ASSERTION(img->fmt == IMG_FMT_I420, "WebM image format is not I420");
941 // Chroma shifts are rounded down as per the decoding examples in the VP8 SDK
942 VideoData::YCbCrBuffer b;
943 b.mPlanes[0].mData = img->planes[0];
944 b.mPlanes[0].mStride = img->stride[0];
945 b.mPlanes[0].mHeight = img->d_h;
946 b.mPlanes[0].mWidth = img->d_w;
947 b.mPlanes[0].mOffset = b.mPlanes[0].mSkip = 0;
949 b.mPlanes[1].mData = img->planes[1];
950 b.mPlanes[1].mStride = img->stride[1];
951 b.mPlanes[1].mHeight = (img->d_h + 1) >> img->y_chroma_shift;
952 b.mPlanes[1].mWidth = (img->d_w + 1) >> img->x_chroma_shift;
953 b.mPlanes[1].mOffset = b.mPlanes[1].mSkip = 0;
955 b.mPlanes[2].mData = img->planes[2];
956 b.mPlanes[2].mStride = img->stride[2];
957 b.mPlanes[2].mHeight = (img->d_h + 1) >> img->y_chroma_shift;
958 b.mPlanes[2].mWidth = (img->d_w + 1) >> img->x_chroma_shift;
959 b.mPlanes[2].mOffset = b.mPlanes[2].mSkip = 0;
961 IntRect picture = ToIntRect(mPicture);
962 if (img->d_w != static_cast<uint32_t>(mInitialFrame.width) ||
963 img->d_h != static_cast<uint32_t>(mInitialFrame.height)) {
964 // Frame size is different from what the container reports. This is legal
965 // in WebM, and we will preserve the ratio of the crop rectangle as it
966 // was reported relative to the picture size reported by the container.
967 picture.x = (mPicture.x * img->d_w) / mInitialFrame.width;
968 picture.y = (mPicture.y * img->d_h) / mInitialFrame.height;
969 picture.width = (img->d_w * mPicture.width) / mInitialFrame.width;
970 picture.height = (img->d_h * mPicture.height) / mInitialFrame.height;
971 }
973 VideoData *v = VideoData::Create(mInfo.mVideo,
974 mDecoder->GetImageContainer(),
975 holder->mOffset,
976 tstamp_usecs,
977 (next_tstamp / NS_PER_USEC) - tstamp_usecs,
978 b,
979 si.is_kf,
980 -1,
981 picture);
982 if (!v) {
983 return false;
984 }
985 parsed++;
986 decoded++;
987 NS_ASSERTION(decoded <= parsed,
988 "Expect only 1 frame per chunk per packet in WebM...");
989 VideoQueue().Push(v);
990 }
991 }
993 return true;
994 }
996 void
997 WebMReader::PushVideoPacket(NesteggPacketHolder* aItem)
998 {
999 mVideoPackets.PushFront(aItem);
1000 }
1002 nsresult WebMReader::Seek(int64_t aTarget, int64_t aStartTime, int64_t aEndTime,
1003 int64_t aCurrentTime)
1004 {
1005 NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
1007 LOG(PR_LOG_DEBUG, ("Reader [%p] for Decoder [%p]: About to seek to %fs",
1008 this, mDecoder, aTarget/1000000.0));
1009 if (NS_FAILED(ResetDecode())) {
1010 return NS_ERROR_FAILURE;
1011 }
1012 uint32_t trackToSeek = mHasVideo ? mVideoTrack : mAudioTrack;
1013 uint64_t target = aTarget * NS_PER_USEC;
1014 if (mSeekPreroll) {
1015 target = std::max(static_cast<uint64_t>(aStartTime * NS_PER_USEC), target - mSeekPreroll);
1016 }
1017 int r = nestegg_track_seek(mContext, trackToSeek, target);
1018 if (r != 0) {
1019 // Try seeking directly based on cluster information in memory.
1020 int64_t offset = 0;
1021 bool rv = mBufferedState->GetOffsetForTime((aTarget - aStartTime)/NS_PER_USEC, &offset);
1022 if (!rv) {
1023 return NS_ERROR_FAILURE;
1024 }
1026 r = nestegg_offset_seek(mContext, offset);
1027 if (r != 0) {
1028 return NS_ERROR_FAILURE;
1029 }
1030 }
1031 return NS_OK;
1032 }
1034 nsresult WebMReader::GetBuffered(dom::TimeRanges* aBuffered, int64_t aStartTime)
1035 {
1036 MediaResource* resource = mDecoder->GetResource();
1038 uint64_t timecodeScale;
1039 if (!mContext || nestegg_tstamp_scale(mContext, &timecodeScale) == -1) {
1040 return NS_OK;
1041 }
1043 // Special case completely cached files. This also handles local files.
1044 bool isFullyCached = resource->IsDataCachedToEndOfResource(0);
1045 if (isFullyCached) {
1046 uint64_t duration = 0;
1047 if (nestegg_duration(mContext, &duration) == 0) {
1048 aBuffered->Add(0, duration / NS_PER_S);
1049 }
1050 }
1052 uint32_t bufferedLength = 0;
1053 aBuffered->GetLength(&bufferedLength);
1055 // Either we the file is not fully cached, or we couldn't find a duration in
1056 // the WebM bitstream.
1057 if (!isFullyCached || !bufferedLength) {
1058 MediaResource* resource = mDecoder->GetResource();
1059 nsTArray<MediaByteRange> ranges;
1060 nsresult res = resource->GetCachedRanges(ranges);
1061 NS_ENSURE_SUCCESS(res, res);
1063 for (uint32_t index = 0; index < ranges.Length(); index++) {
1064 uint64_t start, end;
1065 bool rv = mBufferedState->CalculateBufferedForRange(ranges[index].mStart,
1066 ranges[index].mEnd,
1067 &start, &end);
1068 if (rv) {
1069 double startTime = start * timecodeScale / NS_PER_S - aStartTime;
1070 double endTime = end * timecodeScale / NS_PER_S - aStartTime;
1071 // If this range extends to the end of the file, the true end time
1072 // is the file's duration.
1073 if (resource->IsDataCachedToEndOfResource(ranges[index].mStart)) {
1074 uint64_t duration = 0;
1075 if (nestegg_duration(mContext, &duration) == 0) {
1076 endTime = duration / NS_PER_S;
1077 }
1078 }
1080 aBuffered->Add(startTime, endTime);
1081 }
1082 }
1083 }
1085 return NS_OK;
1086 }
1088 void WebMReader::NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64_t aOffset)
1089 {
1090 mBufferedState->NotifyDataArrived(aBuffer, aLength, aOffset);
1091 }
1093 } // namespace mozilla
