1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/content/media/omx/OMXCodecWrapper.cpp Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,797 @@
1.4 +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
1.5 +/* This Source Code Form is subject to the terms of the Mozilla Public
1.6 + * License, v. 2.0. If a copy of the MPL was not distributed with this file,
1.7 + * You can obtain one at http://mozilla.org/MPL/2.0/. */
1.8 +
1.9 +#include "OMXCodecWrapper.h"
1.10 +#include "OMXCodecDescriptorUtil.h"
1.11 +#include "TrackEncoder.h"
1.12 +
1.13 +#include <binder/ProcessState.h>
1.14 +#include <cutils/properties.h>
1.15 +#include <media/ICrypto.h>
1.16 +#include <media/IOMX.h>
1.17 +#include <OMX_Component.h>
1.18 +#include <stagefright/MediaDefs.h>
1.19 +#include <stagefright/MediaErrors.h>
1.20 +
1.21 +#include "AudioChannelFormat.h"
1.22 +#include <mozilla/Monitor.h>
1.23 +#include "mozilla/layers/GrallocTextureClient.h"
1.24 +
1.25 +using namespace mozilla;
1.26 +using namespace mozilla::gfx;
1.27 +using namespace mozilla::layers;
1.28 +
1.29 +#define ENCODER_CONFIG_BITRATE 2000000 // bps
1.30 +// How many seconds between I-frames.
1.31 +#define ENCODER_CONFIG_I_FRAME_INTERVAL 1
1.32 +// Wait up to 5ms for input buffers.
1.33 +#define INPUT_BUFFER_TIMEOUT_US (5 * 1000ll)
1.34 +// AMR NB kbps
1.35 +#define AMRNB_BITRATE 12200
1.36 +
1.37 +#define CODEC_ERROR(args...) \
1.38 + do { \
1.39 + __android_log_print(ANDROID_LOG_ERROR, "OMXCodecWrapper", ##args); \
1.40 + } while (0)
1.41 +
1.42 +namespace android {
1.43 +
1.44 +OMXAudioEncoder*
1.45 +OMXCodecWrapper::CreateAACEncoder()
1.46 +{
1.47 + nsAutoPtr<OMXAudioEncoder> aac(new OMXAudioEncoder(CodecType::AAC_ENC));
1.48 + // Return the object only when media codec is valid.
1.49 + NS_ENSURE_TRUE(aac->IsValid(), nullptr);
1.50 +
1.51 + return aac.forget();
1.52 +}
1.53 +
1.54 +OMXAudioEncoder*
1.55 +OMXCodecWrapper::CreateAMRNBEncoder()
1.56 +{
1.57 + nsAutoPtr<OMXAudioEncoder> amr(new OMXAudioEncoder(CodecType::AMR_NB_ENC));
1.58 + // Return the object only when media codec is valid.
1.59 + NS_ENSURE_TRUE(amr->IsValid(), nullptr);
1.60 +
1.61 + return amr.forget();
1.62 +}
1.63 +
1.64 +OMXVideoEncoder*
1.65 +OMXCodecWrapper::CreateAVCEncoder()
1.66 +{
1.67 + nsAutoPtr<OMXVideoEncoder> avc(new OMXVideoEncoder(CodecType::AVC_ENC));
1.68 + // Return the object only when media codec is valid.
1.69 + NS_ENSURE_TRUE(avc->IsValid(), nullptr);
1.70 +
1.71 + return avc.forget();
1.72 +}
1.73 +
1.74 +OMXCodecWrapper::OMXCodecWrapper(CodecType aCodecType)
1.75 + : mCodecType(aCodecType)
1.76 + , mStarted(false)
1.77 + , mAMRCSDProvided(false)
1.78 +{
1.79 + ProcessState::self()->startThreadPool();
1.80 +
1.81 + mLooper = new ALooper();
1.82 + mLooper->start();
1.83 +
1.84 + if (aCodecType == CodecType::AVC_ENC) {
1.85 + mCodec = MediaCodec::CreateByType(mLooper, MEDIA_MIMETYPE_VIDEO_AVC, true);
1.86 + } else if (aCodecType == CodecType::AMR_NB_ENC) {
1.87 + mCodec = MediaCodec::CreateByType(mLooper, MEDIA_MIMETYPE_AUDIO_AMR_NB, true);
1.88 + } else if (aCodecType == CodecType::AAC_ENC) {
1.89 + mCodec = MediaCodec::CreateByType(mLooper, MEDIA_MIMETYPE_AUDIO_AAC, true);
1.90 + } else {
1.91 + NS_ERROR("Unknown codec type.");
1.92 + }
1.93 +}
1.94 +
1.95 +OMXCodecWrapper::~OMXCodecWrapper()
1.96 +{
1.97 + if (mCodec.get()) {
1.98 + Stop();
1.99 + mCodec->release();
1.100 + }
1.101 + mLooper->stop();
1.102 +}
1.103 +
1.104 +status_t
1.105 +OMXCodecWrapper::Start()
1.106 +{
1.107 + // Already started.
1.108 + NS_ENSURE_FALSE(mStarted, OK);
1.109 +
1.110 + status_t result = mCodec->start();
1.111 + mStarted = (result == OK);
1.112 +
1.113 + // Get references to MediaCodec buffers.
1.114 + if (result == OK) {
1.115 + mCodec->getInputBuffers(&mInputBufs);
1.116 + mCodec->getOutputBuffers(&mOutputBufs);
1.117 + }
1.118 +
1.119 + return result;
1.120 +}
1.121 +
1.122 +status_t
1.123 +OMXCodecWrapper::Stop()
1.124 +{
1.125 + // Already stopped.
1.126 + NS_ENSURE_TRUE(mStarted, OK);
1.127 +
1.128 + status_t result = mCodec->stop();
1.129 + mStarted = !(result == OK);
1.130 +
1.131 + return result;
1.132 +}
1.133 +
1.134 +// Check system property to see if we're running on emulator.
1.135 +static bool
1.136 +IsRunningOnEmulator()
1.137 +{
1.138 + char qemu[PROPERTY_VALUE_MAX];
1.139 + property_get("ro.kernel.qemu", qemu, "");
1.140 + return strncmp(qemu, "1", 1) == 0;
1.141 +}
1.142 +
1.143 +nsresult
1.144 +OMXVideoEncoder::Configure(int aWidth, int aHeight, int aFrameRate,
1.145 + BlobFormat aBlobFormat)
1.146 +{
1.147 + MOZ_ASSERT(!mStarted, "Configure() was called already.");
1.148 +
1.149 + NS_ENSURE_TRUE(aWidth > 0 && aHeight > 0 && aFrameRate > 0,
1.150 + NS_ERROR_INVALID_ARG);
1.151 +
1.152 + OMX_VIDEO_AVCLEVELTYPE level = OMX_VIDEO_AVCLevel3;
1.153 + OMX_VIDEO_CONTROLRATETYPE bitrateMode = OMX_Video_ControlRateConstant;
1.154 + // Limitation of soft AVC/H.264 encoder running on emulator in stagefright.
1.155 + static bool emu = IsRunningOnEmulator();
1.156 + if (emu) {
1.157 + if (aWidth > 352 || aHeight > 288) {
1.158 + CODEC_ERROR("SoftAVCEncoder doesn't support resolution larger than CIF");
1.159 + return NS_ERROR_INVALID_ARG;
1.160 + }
1.161 + level = OMX_VIDEO_AVCLevel2;
1.162 + bitrateMode = OMX_Video_ControlRateVariable;
1.163 + }
1.164 +
1.165 + // Set up configuration parameters for AVC/H.264 encoder.
1.166 + sp<AMessage> format = new AMessage;
1.167 + // Fixed values
1.168 + format->setString("mime", MEDIA_MIMETYPE_VIDEO_AVC);
1.169 + format->setInt32("bitrate", ENCODER_CONFIG_BITRATE);
1.170 + format->setInt32("i-frame-interval", ENCODER_CONFIG_I_FRAME_INTERVAL);
1.171 + // See mozilla::layers::GrallocImage, supports YUV 4:2:0, CbCr width and
1.172 + // height is half that of Y
1.173 + format->setInt32("color-format", OMX_COLOR_FormatYUV420SemiPlanar);
1.174 + format->setInt32("profile", OMX_VIDEO_AVCProfileBaseline);
1.175 + format->setInt32("level", level);
1.176 + format->setInt32("bitrate-mode", bitrateMode);
1.177 + format->setInt32("store-metadata-in-buffers", 0);
1.178 + format->setInt32("prepend-sps-pps-to-idr-frames", 0);
1.179 + // Input values.
1.180 + format->setInt32("width", aWidth);
1.181 + format->setInt32("height", aHeight);
1.182 + format->setInt32("stride", aWidth);
1.183 + format->setInt32("slice-height", aHeight);
1.184 + format->setInt32("frame-rate", aFrameRate);
1.185 +
1.186 + status_t result = mCodec->configure(format, nullptr, nullptr,
1.187 + MediaCodec::CONFIGURE_FLAG_ENCODE);
1.188 + NS_ENSURE_TRUE(result == OK, NS_ERROR_FAILURE);
1.189 +
1.190 + mWidth = aWidth;
1.191 + mHeight = aHeight;
1.192 + mBlobFormat = aBlobFormat;
1.193 +
1.194 + result = Start();
1.195 +
1.196 + return result == OK ? NS_OK : NS_ERROR_FAILURE;
1.197 +}
1.198 +
1.199 +// Copy pixels from planar YUV (4:4:4/4:2:2/4:2:0) or NV21 (semi-planar 4:2:0)
1.200 +// format to NV12 (semi-planar 4:2:0) format for QCOM HW encoder.
1.201 +// Planar YUV: YYY...UUU...VVV...
1.202 +// NV21: YYY...VUVU...
1.203 +// NV12: YYY...UVUV...
1.204 +// For 4:4:4/4:2:2 -> 4:2:0, subsample using odd row/column without
1.205 +// interpolation.
1.206 +// aSource contains info about source image data, and the result will be stored
1.207 +// in aDestination, whose size needs to be >= Y plane size * 3 / 2.
1.208 +static void
1.209 +ConvertPlanarYCbCrToNV12(const PlanarYCbCrData* aSource, uint8_t* aDestination)
1.210 +{
1.211 + // Fill Y plane.
1.212 + uint8_t* y = aSource->mYChannel;
1.213 + IntSize ySize = aSource->mYSize;
1.214 +
1.215 + // Y plane.
1.216 + for (int i = 0; i < ySize.height; i++) {
1.217 + memcpy(aDestination, y, ySize.width);
1.218 + aDestination += ySize.width;
1.219 + y += aSource->mYStride;
1.220 + }
1.221 +
1.222 + // Fill interleaved UV plane.
1.223 + uint8_t* u = aSource->mCbChannel;
1.224 + uint8_t* v = aSource->mCrChannel;
1.225 + IntSize uvSize = aSource->mCbCrSize;
1.226 + // Subsample to 4:2:0 if source is 4:4:4 or 4:2:2.
1.227 + // Y plane width & height should be multiple of U/V plane width & height.
1.228 + MOZ_ASSERT(ySize.width % uvSize.width == 0 &&
1.229 + ySize.height % uvSize.height == 0);
1.230 + size_t uvWidth = ySize.width / 2;
1.231 + size_t uvHeight = ySize.height / 2;
1.232 + size_t horiSubsample = uvSize.width / uvWidth;
1.233 + size_t uPixStride = horiSubsample * (1 + aSource->mCbSkip);
1.234 + size_t vPixStride = horiSubsample * (1 + aSource->mCrSkip);
1.235 + size_t lineStride = uvSize.height / uvHeight * aSource->mCbCrStride;
1.236 +
1.237 + for (int i = 0; i < uvHeight; i++) {
1.238 + // 1st pixel per line.
1.239 + uint8_t* uSrc = u;
1.240 + uint8_t* vSrc = v;
1.241 + for (int j = 0; j < uvWidth; j++) {
1.242 + *aDestination++ = *uSrc;
1.243 + *aDestination++ = *vSrc;
1.244 + // Pick next source pixel.
1.245 + uSrc += uPixStride;
1.246 + vSrc += vPixStride;
1.247 + }
1.248 + // Pick next source line.
1.249 + u += lineStride;
1.250 + v += lineStride;
1.251 + }
1.252 +}
1.253 +
1.254 +// Convert pixels in graphic buffer to NV12 format. aSource is the layer image
1.255 +// containing source graphic buffer, and aDestination is the destination of
1.256 +// conversion. Currently only 2 source format are supported:
1.257 +// - NV21/HAL_PIXEL_FORMAT_YCrCb_420_SP (from camera preview window).
1.258 +// - YV12/HAL_PIXEL_FORMAT_YV12 (from video decoder).
1.259 +static void
1.260 +ConvertGrallocImageToNV12(GrallocImage* aSource, uint8_t* aDestination)
1.261 +{
1.262 + // Get graphic buffer.
1.263 + sp<GraphicBuffer> graphicBuffer = aSource->GetGraphicBuffer();
1.264 +
1.265 + int pixelFormat = graphicBuffer->getPixelFormat();
1.266 + // Only support NV21 (from camera) or YV12 (from HW decoder output) for now.
1.267 + NS_ENSURE_TRUE_VOID(pixelFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP ||
1.268 + pixelFormat == HAL_PIXEL_FORMAT_YV12);
1.269 +
1.270 + void* imgPtr = nullptr;
1.271 + graphicBuffer->lock(GraphicBuffer::USAGE_SW_READ_MASK, &imgPtr);
1.272 + // Build PlanarYCbCrData for NV21 or YV12 buffer.
1.273 + PlanarYCbCrData yuv;
1.274 + switch (pixelFormat) {
1.275 + case HAL_PIXEL_FORMAT_YCrCb_420_SP: // From camera.
1.276 + yuv.mYChannel = static_cast<uint8_t*>(imgPtr);
1.277 + yuv.mYSkip = 0;
1.278 + yuv.mYSize.width = graphicBuffer->getWidth();
1.279 + yuv.mYSize.height = graphicBuffer->getHeight();
1.280 + yuv.mYStride = graphicBuffer->getStride();
1.281 + // 4:2:0.
1.282 + yuv.mCbCrSize.width = yuv.mYSize.width / 2;
1.283 + yuv.mCbCrSize.height = yuv.mYSize.height / 2;
1.284 + // Interleaved VU plane.
1.285 + yuv.mCrChannel = yuv.mYChannel + (yuv.mYStride * yuv.mYSize.height);
1.286 + yuv.mCrSkip = 1;
1.287 + yuv.mCbChannel = yuv.mCrChannel + 1;
1.288 + yuv.mCbSkip = 1;
1.289 + yuv.mCbCrStride = yuv.mYStride;
1.290 + ConvertPlanarYCbCrToNV12(&yuv, aDestination);
1.291 + break;
1.292 + case HAL_PIXEL_FORMAT_YV12: // From video decoder.
1.293 + // Android YV12 format is defined in system/core/include/system/graphics.h
1.294 + yuv.mYChannel = static_cast<uint8_t*>(imgPtr);
1.295 + yuv.mYSkip = 0;
1.296 + yuv.mYSize.width = graphicBuffer->getWidth();
1.297 + yuv.mYSize.height = graphicBuffer->getHeight();
1.298 + yuv.mYStride = graphicBuffer->getStride();
1.299 + // 4:2:0.
1.300 + yuv.mCbCrSize.width = yuv.mYSize.width / 2;
1.301 + yuv.mCbCrSize.height = yuv.mYSize.height / 2;
1.302 + yuv.mCrChannel = yuv.mYChannel + (yuv.mYStride * yuv.mYSize.height);
1.303 + // Aligned to 16 bytes boundary.
1.304 + yuv.mCbCrStride = (yuv.mYStride / 2 + 15) & ~0x0F;
1.305 + yuv.mCrSkip = 0;
1.306 + yuv.mCbChannel = yuv.mCrChannel + (yuv.mCbCrStride * yuv.mCbCrSize.height);
1.307 + yuv.mCbSkip = 0;
1.308 + ConvertPlanarYCbCrToNV12(&yuv, aDestination);
1.309 + break;
1.310 + default:
1.311 + NS_ERROR("Unsupported input gralloc image type. Should never be here.");
1.312 + }
1.313 +
1.314 + graphicBuffer->unlock();
1.315 +}
1.316 +
1.317 +nsresult
1.318 +OMXVideoEncoder::Encode(const Image* aImage, int aWidth, int aHeight,
1.319 + int64_t aTimestamp, int aInputFlags)
1.320 +{
1.321 + MOZ_ASSERT(mStarted, "Configure() should be called before Encode().");
1.322 +
1.323 + NS_ENSURE_TRUE(aWidth == mWidth && aHeight == mHeight && aTimestamp >= 0,
1.324 + NS_ERROR_INVALID_ARG);
1.325 +
1.326 + status_t result;
1.327 +
1.328 + // Dequeue an input buffer.
1.329 + uint32_t index;
1.330 + result = mCodec->dequeueInputBuffer(&index, INPUT_BUFFER_TIMEOUT_US);
1.331 + NS_ENSURE_TRUE(result == OK, NS_ERROR_FAILURE);
1.332 +
1.333 + const sp<ABuffer>& inBuf = mInputBufs.itemAt(index);
1.334 + uint8_t* dst = inBuf->data();
1.335 + size_t dstSize = inBuf->capacity();
1.336 +
1.337 + size_t yLen = aWidth * aHeight;
1.338 + size_t uvLen = yLen / 2;
1.339 + // Buffer should be large enough to hold input image data.
1.340 + MOZ_ASSERT(dstSize >= yLen + uvLen);
1.341 +
1.342 + inBuf->setRange(0, yLen + uvLen);
1.343 +
1.344 + if (!aImage) {
1.345 + // Generate muted/black image directly in buffer.
1.346 + dstSize = yLen + uvLen;
1.347 + // Fill Y plane.
1.348 + memset(dst, 0x10, yLen);
1.349 + // Fill UV plane.
1.350 + memset(dst + yLen, 0x80, uvLen);
1.351 + } else {
1.352 + Image* img = const_cast<Image*>(aImage);
1.353 + ImageFormat format = img->GetFormat();
1.354 +
1.355 + MOZ_ASSERT(aWidth == img->GetSize().width &&
1.356 + aHeight == img->GetSize().height);
1.357 +
1.358 + if (format == ImageFormat::GRALLOC_PLANAR_YCBCR) {
1.359 + ConvertGrallocImageToNV12(static_cast<GrallocImage*>(img), dst);
1.360 + } else if (format == ImageFormat::PLANAR_YCBCR) {
1.361 + ConvertPlanarYCbCrToNV12(static_cast<PlanarYCbCrImage*>(img)->GetData(),
1.362 + dst);
1.363 + } else {
1.364 + // TODO: support RGB to YUV color conversion.
1.365 + NS_ERROR("Unsupported input image type.");
1.366 + }
1.367 + }
1.368 +
1.369 + // Queue this input buffer.
1.370 + result = mCodec->queueInputBuffer(index, 0, dstSize, aTimestamp, aInputFlags);
1.371 +
1.372 + return result == OK ? NS_OK : NS_ERROR_FAILURE;
1.373 +}
1.374 +
1.375 +status_t
1.376 +OMXVideoEncoder::AppendDecoderConfig(nsTArray<uint8_t>* aOutputBuf,
1.377 + ABuffer* aData)
1.378 +{
1.379 + // Codec already parsed aData. Using its result makes generating config blob
1.380 + // much easier.
1.381 + sp<AMessage> format;
1.382 + mCodec->getOutputFormat(&format);
1.383 +
1.384 + // NAL unit format is needed by WebRTC for RTP packets; AVC/H.264 decoder
1.385 + // config descriptor is needed to construct MP4 'avcC' box.
1.386 + status_t result = GenerateAVCDescriptorBlob(format, aOutputBuf, mBlobFormat);
1.387 + mHasConfigBlob = (result == OK);
1.388 +
1.389 + return result;
1.390 +}
1.391 +
1.392 +// Override to replace NAL unit start code with 4-bytes unit length.
1.393 +// See ISO/IEC 14496-15 5.2.3.
1.394 +void
1.395 +OMXVideoEncoder::AppendFrame(nsTArray<uint8_t>* aOutputBuf,
1.396 + const uint8_t* aData, size_t aSize)
1.397 +{
1.398 + aOutputBuf->SetCapacity(aSize);
1.399 +
1.400 + if (mBlobFormat == BlobFormat::AVC_NAL) {
1.401 + // Append NAL format data without modification.
1.402 + aOutputBuf->AppendElements(aData, aSize);
1.403 + return;
1.404 + }
1.405 + // Replace start code with data length.
1.406 + uint8_t length[] = {
1.407 + (aSize >> 24) & 0xFF,
1.408 + (aSize >> 16) & 0xFF,
1.409 + (aSize >> 8) & 0xFF,
1.410 + aSize & 0xFF,
1.411 + };
1.412 + aOutputBuf->AppendElements(length, sizeof(length));
1.413 + aOutputBuf->AppendElements(aData + sizeof(length), aSize);
1.414 +}
1.415 +
1.416 +nsresult
1.417 +OMXVideoEncoder::GetCodecConfig(nsTArray<uint8_t>* aOutputBuf)
1.418 +{
1.419 + MOZ_ASSERT(mHasConfigBlob, "Haven't received codec config yet.");
1.420 +
1.421 + return AppendDecoderConfig(aOutputBuf, nullptr) == OK ? NS_OK : NS_ERROR_FAILURE;
1.422 +}
1.423 +
1.424 +// MediaCodec::setParameters() is available only after API level 18.
1.425 +#if ANDROID_VERSION >= 18
1.426 +nsresult
1.427 +OMXVideoEncoder::SetBitrate(int32_t aKbps)
1.428 +{
1.429 + sp<AMessage> msg = new AMessage();
1.430 + msg->setInt32("videoBitrate", aKbps * 1000 /* kbps -> bps */);
1.431 + status_t result = mCodec->setParameters(msg);
1.432 + MOZ_ASSERT(result == OK);
1.433 + return result == OK ? NS_OK : NS_ERROR_FAILURE;
1.434 +}
1.435 +#endif
1.436 +
1.437 +nsresult
1.438 +OMXAudioEncoder::Configure(int aChannels, int aInputSampleRate,
1.439 + int aEncodedSampleRate)
1.440 +{
1.441 + MOZ_ASSERT(!mStarted);
1.442 +
1.443 + NS_ENSURE_TRUE(aChannels > 0 && aInputSampleRate > 0 && aEncodedSampleRate >= 0,
1.444 + NS_ERROR_INVALID_ARG);
1.445 +
1.446 + if (aInputSampleRate != aEncodedSampleRate) {
1.447 + int error;
1.448 + mResampler = speex_resampler_init(aChannels,
1.449 + aInputSampleRate,
1.450 + aEncodedSampleRate,
1.451 + SPEEX_RESAMPLER_QUALITY_DEFAULT,
1.452 + &error);
1.453 +
1.454 + if (error != RESAMPLER_ERR_SUCCESS) {
1.455 + return NS_ERROR_FAILURE;
1.456 + }
1.457 + speex_resampler_skip_zeros(mResampler);
1.458 + }
1.459 + // Set up configuration parameters for AAC encoder.
1.460 + sp<AMessage> format = new AMessage;
1.461 + // Fixed values.
1.462 + if (mCodecType == AAC_ENC) {
1.463 + format->setString("mime", MEDIA_MIMETYPE_AUDIO_AAC);
1.464 + format->setInt32("aac-profile", OMX_AUDIO_AACObjectLC);
1.465 + format->setInt32("bitrate", kAACBitrate);
1.466 + format->setInt32("sample-rate", aInputSampleRate);
1.467 + } else if (mCodecType == AMR_NB_ENC) {
1.468 + format->setString("mime", MEDIA_MIMETYPE_AUDIO_AMR_NB);
1.469 + format->setInt32("bitrate", AMRNB_BITRATE);
1.470 + format->setInt32("sample-rate", aEncodedSampleRate);
1.471 + } else {
1.472 + MOZ_ASSERT(false, "Can't support this codec type!!");
1.473 + }
1.474 + // Input values.
1.475 + format->setInt32("channel-count", aChannels);
1.476 +
1.477 + status_t result = mCodec->configure(format, nullptr, nullptr,
1.478 + MediaCodec::CONFIGURE_FLAG_ENCODE);
1.479 + NS_ENSURE_TRUE(result == OK, NS_ERROR_FAILURE);
1.480 +
1.481 + mChannels = aChannels;
1.482 + mSampleDuration = 1000000 / aInputSampleRate;
1.483 + mResamplingRatio = aEncodedSampleRate > 0 ? 1.0 *
1.484 + aEncodedSampleRate / aInputSampleRate : 1.0;
1.485 + result = Start();
1.486 +
1.487 + return result == OK ? NS_OK : NS_ERROR_FAILURE;
1.488 +}
1.489 +
1.490 +class InputBufferHelper MOZ_FINAL {
1.491 +public:
1.492 + InputBufferHelper(sp<MediaCodec>& aCodec, Vector<sp<ABuffer> >& aBuffers)
1.493 + : mCodec(aCodec)
1.494 + , mBuffers(aBuffers)
1.495 + , mIndex(0)
1.496 + , mData(nullptr)
1.497 + , mOffset(0)
1.498 + , mCapicity(0)
1.499 + {}
1.500 +
1.501 + ~InputBufferHelper()
1.502 + {
1.503 + // Unflushed data in buffer.
1.504 + MOZ_ASSERT(!mData);
1.505 + }
1.506 +
1.507 + status_t Dequeue()
1.508 + {
1.509 + // Shouldn't have dequeued buffer.
1.510 + MOZ_ASSERT(!mData);
1.511 +
1.512 + status_t result = mCodec->dequeueInputBuffer(&mIndex,
1.513 + INPUT_BUFFER_TIMEOUT_US);
1.514 + NS_ENSURE_TRUE(result == OK, result);
1.515 + sp<ABuffer> inBuf = mBuffers.itemAt(mIndex);
1.516 + mData = inBuf->data();
1.517 + mCapicity = inBuf->capacity();
1.518 + mOffset = 0;
1.519 +
1.520 + return OK;
1.521 + }
1.522 +
1.523 + uint8_t* GetPointer() { return mData + mOffset; }
1.524 +
1.525 + const size_t AvailableSize() { return mCapicity - mOffset; }
1.526 +
1.527 + void IncreaseOffset(size_t aValue)
1.528 + {
1.529 + // Should never out of bound.
1.530 + MOZ_ASSERT(mOffset + aValue <= mCapicity);
1.531 + mOffset += aValue;
1.532 + }
1.533 +
1.534 + status_t Enqueue(int64_t aTimestamp, int aFlags)
1.535 + {
1.536 + // Should have dequeued buffer.
1.537 + MOZ_ASSERT(mData);
1.538 +
1.539 + // Queue this buffer.
1.540 + status_t result = mCodec->queueInputBuffer(mIndex, 0, mOffset, aTimestamp,
1.541 + aFlags);
1.542 + NS_ENSURE_TRUE(result == OK, result);
1.543 + mData = nullptr;
1.544 +
1.545 + return OK;
1.546 + }
1.547 +
1.548 +private:
1.549 + sp<MediaCodec>& mCodec;
1.550 + Vector<sp<ABuffer> >& mBuffers;
1.551 + size_t mIndex;
1.552 + uint8_t* mData;
1.553 + size_t mCapicity;
1.554 + size_t mOffset;
1.555 +};
1.556 +
1.557 +OMXAudioEncoder::~OMXAudioEncoder()
1.558 +{
1.559 + if (mResampler) {
1.560 + speex_resampler_destroy(mResampler);
1.561 + mResampler = nullptr;
1.562 + }
1.563 +}
1.564 +
1.565 +nsresult
1.566 +OMXAudioEncoder::Encode(AudioSegment& aSegment, int aInputFlags)
1.567 +{
1.568 +#ifndef MOZ_SAMPLE_TYPE_S16
1.569 +#error MediaCodec accepts only 16-bit PCM data.
1.570 +#endif
1.571 +
1.572 + MOZ_ASSERT(mStarted, "Configure() should be called before Encode().");
1.573 +
1.574 + size_t numSamples = aSegment.GetDuration();
1.575 +
1.576 + // Get input buffer.
1.577 + InputBufferHelper buffer(mCodec, mInputBufs);
1.578 + status_t result = buffer.Dequeue();
1.579 + if (result == -EAGAIN) {
1.580 + // All input buffers are full. Caller can try again later after consuming
1.581 + // some output buffers.
1.582 + return NS_OK;
1.583 + }
1.584 + NS_ENSURE_TRUE(result == OK, NS_ERROR_FAILURE);
1.585 +
1.586 + size_t sourceSamplesCopied = 0; // Number of copied samples.
1.587 +
1.588 + if (numSamples > 0) {
1.589 + // Copy input PCM data to input buffer until queue is empty.
1.590 + AudioSegment::ChunkIterator iter(const_cast<AudioSegment&>(aSegment));
1.591 + while (!iter.IsEnded()) {
1.592 + AudioChunk chunk = *iter;
1.593 + size_t sourceSamplesToCopy = chunk.GetDuration(); // Number of samples to copy.
1.594 + size_t bytesToCopy = sourceSamplesToCopy * mChannels *
1.595 + sizeof(AudioDataValue) * mResamplingRatio;
1.596 + if (bytesToCopy > buffer.AvailableSize()) {
1.597 + // Not enough space left in input buffer. Send it to encoder and get a
1.598 + // new one.
1.599 + result = buffer.Enqueue(mTimestamp, aInputFlags & ~BUFFER_EOS);
1.600 + NS_ENSURE_TRUE(result == OK, NS_ERROR_FAILURE);
1.601 +
1.602 + result = buffer.Dequeue();
1.603 + if (result == -EAGAIN) {
1.604 + // All input buffers are full. Caller can try again later after
1.605 + // consuming some output buffers.
1.606 + aSegment.RemoveLeading(sourceSamplesCopied);
1.607 + return NS_OK;
1.608 + }
1.609 +
1.610 + mTimestamp += sourceSamplesCopied * mSampleDuration;
1.611 + sourceSamplesCopied = 0;
1.612 +
1.613 + NS_ENSURE_TRUE(result == OK, NS_ERROR_FAILURE);
1.614 + }
1.615 +
1.616 + AudioDataValue* dst = reinterpret_cast<AudioDataValue*>(buffer.GetPointer());
1.617 + uint32_t dstSamplesCopied = sourceSamplesToCopy;
1.618 + if (!chunk.IsNull()) {
1.619 + if (mResampler) {
1.620 + nsAutoTArray<AudioDataValue, 9600> pcm;
1.621 + pcm.SetLength(bytesToCopy);
1.622 + // Append the interleaved data to input buffer.
1.623 + AudioTrackEncoder::InterleaveTrackData(chunk, sourceSamplesToCopy,
1.624 + mChannels,
1.625 + pcm.Elements());
1.626 + uint32_t inframes = sourceSamplesToCopy;
1.627 + short* in = reinterpret_cast<short*>(pcm.Elements());
1.628 + speex_resampler_process_interleaved_int(mResampler, in, &inframes,
1.629 + dst, &dstSamplesCopied);
1.630 + } else {
1.631 + AudioTrackEncoder::InterleaveTrackData(chunk, sourceSamplesToCopy,
1.632 + mChannels,
1.633 + dst);
1.634 + dstSamplesCopied = sourceSamplesToCopy * mChannels;
1.635 + }
1.636 + } else {
1.637 + // Silence.
1.638 + memset(dst, 0, mResamplingRatio * sourceSamplesToCopy * sizeof(AudioDataValue));
1.639 + }
1.640 +
1.641 + sourceSamplesCopied += sourceSamplesToCopy;
1.642 + buffer.IncreaseOffset(dstSamplesCopied * sizeof(AudioDataValue));
1.643 + iter.Next();
1.644 + }
1.645 + if (sourceSamplesCopied > 0) {
1.646 + aSegment.RemoveLeading(sourceSamplesCopied);
1.647 + }
1.648 + } else if (aInputFlags & BUFFER_EOS) {
1.649 + // No audio data left in segment but we still have to feed something to
1.650 + // MediaCodec in order to notify EOS.
1.651 + size_t bytesToCopy = mChannels * sizeof(AudioDataValue);
1.652 + memset(buffer.GetPointer(), 0, bytesToCopy);
1.653 + buffer.IncreaseOffset(bytesToCopy);
1.654 + sourceSamplesCopied = 1;
1.655 + }
1.656 +
1.657 + if (sourceSamplesCopied > 0) {
1.658 + int flags = aInputFlags;
1.659 + if (aSegment.GetDuration() > 0) {
1.660 + // Don't signal EOS until source segment is empty.
1.661 + flags &= ~BUFFER_EOS;
1.662 + }
1.663 + result = buffer.Enqueue(mTimestamp, flags);
1.664 + NS_ENSURE_TRUE(result == OK, NS_ERROR_FAILURE);
1.665 +
1.666 + mTimestamp += sourceSamplesCopied * mSampleDuration;
1.667 + }
1.668 +
1.669 + return NS_OK;
1.670 +}
1.671 +
1.672 +// Generate decoder config descriptor (defined in ISO/IEC 14496-1 8.3.4.1) for
1.673 +// AAC. The hard-coded bytes are copied from
1.674 +// MPEG4Writer::Track::writeMp4aEsdsBox() implementation in libstagefright.
1.675 +status_t
1.676 +OMXAudioEncoder::AppendDecoderConfig(nsTArray<uint8_t>* aOutputBuf,
1.677 + ABuffer* aData)
1.678 +{
1.679 + MOZ_ASSERT(aData);
1.680 +
1.681 + const size_t csdSize = aData->size();
1.682 +
1.683 + // See
1.684 + // http://wiki.multimedia.cx/index.php?title=Understanding_AAC#Packaging.2FEncapsulation_And_Setup_Data
1.685 + // AAC decoder specific descriptor contains 2 bytes.
1.686 + NS_ENSURE_TRUE(csdSize == 2, ERROR_MALFORMED);
1.687 + // Encoder output must be consistent with kAACFrameDuration:
1.688 + // 14th bit (frame length flag) == 0 => 1024 (kAACFrameDuration) samples.
1.689 + NS_ENSURE_TRUE((aData->data()[1] & 0x04) == 0, ERROR_MALFORMED);
1.690 +
1.691 + // Decoder config descriptor
1.692 + const uint8_t decConfig[] = {
1.693 + 0x04, // Decoder config descriptor tag.
1.694 + 15 + csdSize, // Size: following bytes + csd size.
1.695 + 0x40, // Object type: MPEG-4 audio.
1.696 + 0x15, // Stream type: audio, reserved: 1.
1.697 + 0x00, 0x03, 0x00, // Buffer size: 768 (kAACFrameSize).
1.698 + 0x00, 0x01, 0x77, 0x00, // Max bitrate: 96000 (kAACBitrate).
1.699 + 0x00, 0x01, 0x77, 0x00, // Avg bitrate: 96000 (kAACBitrate).
1.700 + 0x05, // Decoder specific descriptor tag.
1.701 + csdSize, // Data size.
1.702 + };
1.703 + // SL config descriptor.
1.704 + const uint8_t slConfig[] = {
1.705 + 0x06, // SL config descriptor tag.
1.706 + 0x01, // Size.
1.707 + 0x02, // Fixed value.
1.708 + };
1.709 +
1.710 + aOutputBuf->SetCapacity(sizeof(decConfig) + csdSize + sizeof(slConfig));
1.711 + aOutputBuf->AppendElements(decConfig, sizeof(decConfig));
1.712 + aOutputBuf->AppendElements(aData->data(), csdSize);
1.713 + aOutputBuf->AppendElements(slConfig, sizeof(slConfig));
1.714 +
1.715 + return OK;
1.716 +}
1.717 +
1.718 +nsresult
1.719 +OMXCodecWrapper::GetNextEncodedFrame(nsTArray<uint8_t>* aOutputBuf,
1.720 + int64_t* aOutputTimestamp,
1.721 + int* aOutputFlags, int64_t aTimeOut)
1.722 +{
1.723 + MOZ_ASSERT(mStarted,
1.724 + "Configure() should be called before GetNextEncodedFrame().");
1.725 +
1.726 + // Dequeue a buffer from output buffers.
1.727 + size_t index = 0;
1.728 + size_t outOffset = 0;
1.729 + size_t outSize = 0;
1.730 + int64_t outTimeUs = 0;
1.731 + uint32_t outFlags = 0;
1.732 + bool retry = false;
1.733 + do {
1.734 + status_t result = mCodec->dequeueOutputBuffer(&index, &outOffset, &outSize,
1.735 + &outTimeUs, &outFlags,
1.736 + aTimeOut);
1.737 + switch (result) {
1.738 + case OK:
1.739 + break;
1.740 + case INFO_OUTPUT_BUFFERS_CHANGED:
1.741 + // Update our references to new buffers.
1.742 + result = mCodec->getOutputBuffers(&mOutputBufs);
1.743 + // Get output from a new buffer.
1.744 + retry = true;
1.745 + break;
1.746 + case INFO_FORMAT_CHANGED:
1.747 + // It's okay: for encoder, MediaCodec reports this only to inform caller
1.748 + // that there will be a codec config buffer next.
1.749 + return NS_OK;
1.750 + case -EAGAIN:
1.751 + // Output buffer not available. Caller can try again later.
1.752 + return NS_OK;
1.753 + default:
1.754 + CODEC_ERROR("MediaCodec error:%d", result);
1.755 + MOZ_ASSERT(false, "MediaCodec error.");
1.756 + return NS_ERROR_FAILURE;
1.757 + }
1.758 + } while (retry);
1.759 +
1.760 + if (aOutputBuf) {
1.761 + aOutputBuf->Clear();
1.762 + const sp<ABuffer> omxBuf = mOutputBufs.itemAt(index);
1.763 + if (outFlags & MediaCodec::BUFFER_FLAG_CODECCONFIG) {
1.764 + // Codec specific data.
1.765 + if (AppendDecoderConfig(aOutputBuf, omxBuf.get()) != OK) {
1.766 + mCodec->releaseOutputBuffer(index);
1.767 + return NS_ERROR_FAILURE;
1.768 + }
1.769 + } else if ((mCodecType == AMR_NB_ENC) && !mAMRCSDProvided){
1.770 + // OMX AMR codec won't provide csd data, need to generate a fake one.
1.771 + nsRefPtr<EncodedFrame> audiodata = new EncodedFrame();
1.772 + // Decoder config descriptor
1.773 + const uint8_t decConfig[] = {
1.774 + 0x0, 0x0, 0x0, 0x0, // vendor: 4 bytes
1.775 + 0x0, // decoder version
1.776 + 0x83, 0xFF, // mode set: all enabled
1.777 + 0x00, // mode change period
1.778 + 0x01, // frames per sample
1.779 + };
1.780 + aOutputBuf->AppendElements(decConfig, sizeof(decConfig));
1.781 + outFlags |= MediaCodec::BUFFER_FLAG_CODECCONFIG;
1.782 + mAMRCSDProvided = true;
1.783 + } else {
1.784 + AppendFrame(aOutputBuf, omxBuf->data(), omxBuf->size());
1.785 + }
1.786 + }
1.787 + mCodec->releaseOutputBuffer(index);
1.788 +
1.789 + if (aOutputTimestamp) {
1.790 + *aOutputTimestamp = outTimeUs;
1.791 + }
1.792 +
1.793 + if (aOutputFlags) {
1.794 + *aOutputFlags = outFlags;
1.795 + }
1.796 +
1.797 + return NS_OK;
1.798 +}
1.799 +
1.800 +}
