michael@0: /*
michael@0:  * Copyright (C) 2010, Google Inc. All rights reserved.
michael@0:  *
michael@0:  * Redistribution and use in source and binary forms, with or without
michael@0:  * modification, are permitted provided that the following conditions
michael@0:  * are met:
michael@0:  * 1.  Redistributions of source code must retain the above copyright
michael@0:  *    notice, this list of conditions and the following disclaimer.
michael@0:  * 2.  Redistributions in binary form must reproduce the above copyright
michael@0:  *    notice, this list of conditions and the following disclaimer in the
michael@0:  *    documentation and/or other materials provided with the distribution.
michael@0:  *
michael@0:  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY
michael@0:  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
michael@0:  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
michael@0:  * DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
michael@0:  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
michael@0:  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
michael@0:  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
michael@0:  * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
michael@0:  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
michael@0:  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
michael@0:  */
michael@0: 
michael@0: #include "HRTFPanner.h"
michael@0: #include "HRTFDatabaseLoader.h"
michael@0: 
michael@0: #include "FFTConvolver.h"
michael@0: #include "HRTFDatabase.h"
michael@0: 
michael@0: using namespace std;
michael@0: using namespace mozilla;
michael@0: using dom::ChannelInterpretation;
michael@0: 
michael@0: namespace WebCore {
michael@0: 
michael@0: // The value of 2 milliseconds is larger than the largest delay which exists in any HRTFKernel from the default HRTFDatabase (0.0136 seconds).
michael@0: // We ASSERT the delay values used in process() with this value.
michael@0: const double MaxDelayTimeSeconds = 0.002;
michael@0: 
michael@0: const int UninitializedAzimuth = -1;
michael@0: const unsigned RenderingQuantum = WEBAUDIO_BLOCK_SIZE;
michael@0: 
michael@0: HRTFPanner::HRTFPanner(float sampleRate, mozilla::TemporaryRef<HRTFDatabaseLoader> databaseLoader)
michael@0:     : m_databaseLoader(databaseLoader)
michael@0:     , m_sampleRate(sampleRate)
michael@0:     , m_crossfadeSelection(CrossfadeSelection1)
michael@0:     , m_azimuthIndex1(UninitializedAzimuth)
michael@0:     , m_azimuthIndex2(UninitializedAzimuth)
michael@0:     // m_elevation1 and m_elevation2 are initialized in pan()
michael@0:     , m_crossfadeX(0)
michael@0:     , m_crossfadeIncr(0)
michael@0:     , m_convolverL1(HRTFElevation::fftSizeForSampleRate(sampleRate))
michael@0:     , m_convolverR1(m_convolverL1.fftSize())
michael@0:     , m_convolverL2(m_convolverL1.fftSize())
michael@0:     , m_convolverR2(m_convolverL1.fftSize())
michael@0:     , m_delayLine(MaxDelayTimeSeconds * sampleRate, 1.0)
michael@0: {
michael@0:     MOZ_ASSERT(m_databaseLoader);
michael@0:     MOZ_COUNT_CTOR(HRTFPanner);
michael@0: 
michael@0:     m_tempL1.SetLength(RenderingQuantum);
michael@0:     m_tempR1.SetLength(RenderingQuantum);
michael@0:     m_tempL2.SetLength(RenderingQuantum);
michael@0:     m_tempR2.SetLength(RenderingQuantum);
michael@0: }
michael@0: 
michael@0: HRTFPanner::~HRTFPanner()
michael@0: {
michael@0:     MOZ_COUNT_DTOR(HRTFPanner);
michael@0: }
michael@0: 
michael@0: size_t HRTFPanner::sizeOfIncludingThis(mozilla::MallocSizeOf aMallocSizeOf) const
michael@0: {
michael@0:     size_t amount = aMallocSizeOf(this);
michael@0: 
michael@0:     if (m_databaseLoader) {
michael@0:         m_databaseLoader->sizeOfIncludingThis(aMallocSizeOf);
michael@0:     }
michael@0: 
michael@0:     amount += m_convolverL1.sizeOfExcludingThis(aMallocSizeOf);
michael@0:     amount += m_convolverR1.sizeOfExcludingThis(aMallocSizeOf);
michael@0:     amount += m_convolverL2.sizeOfExcludingThis(aMallocSizeOf);
michael@0:     amount += m_convolverR2.sizeOfExcludingThis(aMallocSizeOf);
michael@0:     amount += m_delayLine.SizeOfExcludingThis(aMallocSizeOf);
michael@0:     amount += m_tempL1.SizeOfExcludingThis(aMallocSizeOf);
michael@0:     amount += m_tempL2.SizeOfExcludingThis(aMallocSizeOf);
michael@0:     amount += m_tempR1.SizeOfExcludingThis(aMallocSizeOf);
michael@0:     amount += m_tempR2.SizeOfExcludingThis(aMallocSizeOf);
michael@0: 
michael@0:     return amount;
michael@0: }
michael@0: 
michael@0: void HRTFPanner::reset()
michael@0: {
michael@0:     m_azimuthIndex1 = UninitializedAzimuth;
michael@0:     m_azimuthIndex2 = UninitializedAzimuth;
michael@0:     // m_elevation1 and m_elevation2 are initialized in pan()
michael@0:     m_crossfadeSelection = CrossfadeSelection1;
michael@0:     m_crossfadeX = 0.0f;
michael@0:     m_crossfadeIncr = 0.0f;
michael@0:     m_convolverL1.reset();
michael@0:     m_convolverR1.reset();
michael@0:     m_convolverL2.reset();
michael@0:     m_convolverR2.reset();
michael@0:     m_delayLine.Reset();
michael@0: }
michael@0: 
michael@0: int HRTFPanner::calculateDesiredAzimuthIndexAndBlend(double azimuth, double& azimuthBlend)
michael@0: {
michael@0:     // Convert the azimuth angle from the range -180 -> +180 into the range 0 -> 360.
michael@0:     // The azimuth index may then be calculated from this positive value.
michael@0:     if (azimuth < 0)
michael@0:         azimuth += 360.0;
michael@0: 
michael@0:     HRTFDatabase* database = m_databaseLoader->database();
michael@0:     MOZ_ASSERT(database);
michael@0: 
michael@0:     int numberOfAzimuths = database->numberOfAzimuths();
michael@0:     const double angleBetweenAzimuths = 360.0 / numberOfAzimuths;
michael@0: 
michael@0:     // Calculate the azimuth index and the blend (0 -> 1) for interpolation.
michael@0:     double desiredAzimuthIndexFloat = azimuth / angleBetweenAzimuths;
michael@0:     int desiredAzimuthIndex = static_cast<int>(desiredAzimuthIndexFloat);
michael@0:     azimuthBlend = desiredAzimuthIndexFloat - static_cast<double>(desiredAzimuthIndex);
michael@0: 
michael@0:     // We don't immediately start using this azimuth index, but instead approach this index from the last index we rendered at.
michael@0:     // This minimizes the clicks and graininess for moving sources which occur otherwise.
michael@0:     desiredAzimuthIndex = max(0, desiredAzimuthIndex);
michael@0:     desiredAzimuthIndex = min(numberOfAzimuths - 1, desiredAzimuthIndex);
michael@0:     return desiredAzimuthIndex;
michael@0: }
michael@0: 
michael@0: void HRTFPanner::pan(double desiredAzimuth, double elevation, const AudioChunk* inputBus, AudioChunk* outputBus)
michael@0: {
michael@0: #ifdef DEBUG
michael@0:     unsigned numInputChannels =
michael@0:         inputBus->IsNull() ? 0 : inputBus->mChannelData.Length();
michael@0: 
michael@0:     MOZ_ASSERT(numInputChannels <= 2);
michael@0:     MOZ_ASSERT(inputBus->mDuration == WEBAUDIO_BLOCK_SIZE);
michael@0: #endif
michael@0: 
michael@0:     bool isOutputGood = outputBus && outputBus->mChannelData.Length() == 2 && outputBus->mDuration == WEBAUDIO_BLOCK_SIZE;
michael@0:     MOZ_ASSERT(isOutputGood);
michael@0: 
michael@0:     if (!isOutputGood) {
michael@0:         if (outputBus)
michael@0:             outputBus->SetNull(outputBus->mDuration);
michael@0:         return;
michael@0:     }
michael@0: 
michael@0:     HRTFDatabase* database = m_databaseLoader->database();
michael@0:     if (!database) { // not yet loaded
michael@0:         outputBus->SetNull(outputBus->mDuration);
michael@0:         return;
michael@0:     }
michael@0: 
michael@0:     // IRCAM HRTF azimuths values from the loaded database is reversed from the panner's notion of azimuth.
michael@0:     double azimuth = -desiredAzimuth;
michael@0: 
michael@0:     bool isAzimuthGood = azimuth >= -180.0 && azimuth <= 180.0;
michael@0:     MOZ_ASSERT(isAzimuthGood);
michael@0:     if (!isAzimuthGood) {
michael@0:         outputBus->SetNull(outputBus->mDuration);
michael@0:         return;
michael@0:     }
michael@0: 
michael@0:     // Normally, we'll just be dealing with mono sources.
michael@0:     // If we have a stereo input, implement stereo panning with left source processed by left HRTF, and right source by right HRTF.
michael@0: 
michael@0:     // Get destination pointers.
michael@0:     float* destinationL =
michael@0:         static_cast<float*>(const_cast<void*>(outputBus->mChannelData[0]));
michael@0:     float* destinationR =
michael@0:         static_cast<float*>(const_cast<void*>(outputBus->mChannelData[1]));
michael@0: 
michael@0:     double azimuthBlend;
michael@0:     int desiredAzimuthIndex = calculateDesiredAzimuthIndexAndBlend(azimuth, azimuthBlend);
michael@0: 
michael@0:     // Initially snap azimuth and elevation values to first values encountered.
michael@0:     if (m_azimuthIndex1 == UninitializedAzimuth) {
michael@0:         m_azimuthIndex1 = desiredAzimuthIndex;
michael@0:         m_elevation1 = elevation;
michael@0:     }
michael@0:     if (m_azimuthIndex2 == UninitializedAzimuth) {
michael@0:         m_azimuthIndex2 = desiredAzimuthIndex;
michael@0:         m_elevation2 = elevation;
michael@0:     }
michael@0: 
michael@0:     // Cross-fade / transition over a period of around 45 milliseconds.
michael@0:     // This is an empirical value tuned to be a reasonable trade-off between
michael@0:     // smoothness and speed.
michael@0:     const double fadeFrames = sampleRate() <= 48000 ? 2048 : 4096;
michael@0: 
michael@0:     // Check for azimuth and elevation changes, initiating a cross-fade if needed.
michael@0:     if (!m_crossfadeX && m_crossfadeSelection == CrossfadeSelection1) {
michael@0:         if (desiredAzimuthIndex != m_azimuthIndex1 || elevation != m_elevation1) {
michael@0:             // Cross-fade from 1 -> 2
michael@0:             m_crossfadeIncr = 1 / fadeFrames;
michael@0:             m_azimuthIndex2 = desiredAzimuthIndex;
michael@0:             m_elevation2 = elevation;
michael@0:         }
michael@0:     }
michael@0:     if (m_crossfadeX == 1 && m_crossfadeSelection == CrossfadeSelection2) {
michael@0:         if (desiredAzimuthIndex != m_azimuthIndex2 || elevation != m_elevation2) {
michael@0:             // Cross-fade from 2 -> 1
michael@0:             m_crossfadeIncr = -1 / fadeFrames;
michael@0:             m_azimuthIndex1 = desiredAzimuthIndex;
michael@0:             m_elevation1 = elevation;
michael@0:         }
michael@0:     }
michael@0: 
michael@0:     // Get the HRTFKernels and interpolated delays.
michael@0:     HRTFKernel* kernelL1;
michael@0:     HRTFKernel* kernelR1;
michael@0:     HRTFKernel* kernelL2;
michael@0:     HRTFKernel* kernelR2;
michael@0:     double frameDelayL1;
michael@0:     double frameDelayR1;
michael@0:     double frameDelayL2;
michael@0:     double frameDelayR2;
michael@0:     database->getKernelsFromAzimuthElevation(azimuthBlend, m_azimuthIndex1, m_elevation1, kernelL1, kernelR1, frameDelayL1, frameDelayR1);
michael@0:     database->getKernelsFromAzimuthElevation(azimuthBlend, m_azimuthIndex2, m_elevation2, kernelL2, kernelR2, frameDelayL2, frameDelayR2);
michael@0: 
michael@0:     bool areKernelsGood = kernelL1 && kernelR1 && kernelL2 && kernelR2;
michael@0:     MOZ_ASSERT(areKernelsGood);
michael@0:     if (!areKernelsGood) {
michael@0:         outputBus->SetNull(outputBus->mDuration);
michael@0:         return;
michael@0:     }
michael@0: 
michael@0:     MOZ_ASSERT(frameDelayL1 / sampleRate() < MaxDelayTimeSeconds && frameDelayR1 / sampleRate() < MaxDelayTimeSeconds);
michael@0:     MOZ_ASSERT(frameDelayL2 / sampleRate() < MaxDelayTimeSeconds && frameDelayR2 / sampleRate() < MaxDelayTimeSeconds);
michael@0: 
michael@0:     // Crossfade inter-aural delays based on transitions.
michael@0:     double frameDelaysL[WEBAUDIO_BLOCK_SIZE];
michael@0:     double frameDelaysR[WEBAUDIO_BLOCK_SIZE];
michael@0:     {
michael@0:       float x = m_crossfadeX;
michael@0:       float incr = m_crossfadeIncr;
michael@0:       for (unsigned i = 0; i < WEBAUDIO_BLOCK_SIZE; ++i) {
michael@0:         frameDelaysL[i] = (1 - x) * frameDelayL1 + x * frameDelayL2;
michael@0:         frameDelaysR[i] = (1 - x) * frameDelayR1 + x * frameDelayR2;
michael@0:         x += incr;
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     // First run through delay lines for inter-aural time difference.
michael@0:     m_delayLine.Write(*inputBus);
michael@0:     // "Speakers" means a mono input is read into both outputs (with possibly
michael@0:     // different delays).
michael@0:     m_delayLine.ReadChannel(frameDelaysL, outputBus, 0,
michael@0:                             ChannelInterpretation::Speakers);
michael@0:     m_delayLine.ReadChannel(frameDelaysR, outputBus, 1,
michael@0:                             ChannelInterpretation::Speakers);
michael@0:     m_delayLine.NextBlock();
michael@0: 
michael@0:     bool needsCrossfading = m_crossfadeIncr;
michael@0: 
michael@0:     // Have the convolvers render directly to the final destination if we're not cross-fading.
michael@0:     float* convolutionDestinationL1 = needsCrossfading ? m_tempL1.Elements() : destinationL;
michael@0:     float* convolutionDestinationR1 = needsCrossfading ? m_tempR1.Elements() : destinationR;
michael@0:     float* convolutionDestinationL2 = needsCrossfading ? m_tempL2.Elements() : destinationL;
michael@0:     float* convolutionDestinationR2 = needsCrossfading ? m_tempR2.Elements() : destinationR;
michael@0: 
michael@0:     // Now do the convolutions.
michael@0:     // Note that we avoid doing convolutions on both sets of convolvers if we're not currently cross-fading.
michael@0: 
michael@0:     if (m_crossfadeSelection == CrossfadeSelection1 || needsCrossfading) {
michael@0:         m_convolverL1.process(kernelL1->fftFrame(), destinationL, convolutionDestinationL1, WEBAUDIO_BLOCK_SIZE);
michael@0:         m_convolverR1.process(kernelR1->fftFrame(), destinationR, convolutionDestinationR1, WEBAUDIO_BLOCK_SIZE);
michael@0:     }
michael@0: 
michael@0:     if (m_crossfadeSelection == CrossfadeSelection2 || needsCrossfading) {
michael@0:         m_convolverL2.process(kernelL2->fftFrame(), destinationL, convolutionDestinationL2, WEBAUDIO_BLOCK_SIZE);
michael@0:         m_convolverR2.process(kernelR2->fftFrame(), destinationR, convolutionDestinationR2, WEBAUDIO_BLOCK_SIZE);
michael@0:     }
michael@0: 
michael@0:     if (needsCrossfading) {
michael@0:         // Apply linear cross-fade.
michael@0:         float x = m_crossfadeX;
michael@0:         float incr = m_crossfadeIncr;
michael@0:         for (unsigned i = 0; i < WEBAUDIO_BLOCK_SIZE; ++i) {
michael@0:             destinationL[i] = (1 - x) * convolutionDestinationL1[i] + x * convolutionDestinationL2[i];
michael@0:             destinationR[i] = (1 - x) * convolutionDestinationR1[i] + x * convolutionDestinationR2[i];
michael@0:             x += incr;
michael@0:         }
michael@0:         // Update cross-fade value from local.
michael@0:         m_crossfadeX = x;
michael@0: 
michael@0:         if (m_crossfadeIncr > 0 && fabs(m_crossfadeX - 1) < m_crossfadeIncr) {
michael@0:             // We've fully made the crossfade transition from 1 -> 2.
michael@0:             m_crossfadeSelection = CrossfadeSelection2;
michael@0:             m_crossfadeX = 1;
michael@0:             m_crossfadeIncr = 0;
michael@0:         } else if (m_crossfadeIncr < 0 && fabs(m_crossfadeX) < -m_crossfadeIncr) {
michael@0:             // We've fully made the crossfade transition from 2 -> 1.
michael@0:             m_crossfadeSelection = CrossfadeSelection1;
michael@0:             m_crossfadeX = 0;
michael@0:             m_crossfadeIncr = 0;
michael@0:         }
michael@0:     }
michael@0: }
michael@0: 
michael@0: int HRTFPanner::maxTailFrames() const
michael@0: {
michael@0:     // Although the ideal tail time would be the length of the impulse
michael@0:     // response, there is additional tail time from the approximations in the
michael@0:     // implementation.  Because HRTFPanner is implemented with a DelayKernel
michael@0:     // and a FFTConvolver, the tailTime of the HRTFPanner is the sum of the
michael@0:     // tailTime of the DelayKernel and the tailTime of the FFTConvolver.
michael@0:     // The FFTConvolver has a tail time of fftSize(), including latency of
michael@0:     // fftSize()/2.
michael@0:     return m_delayLine.MaxDelayTicks() + fftSize();
michael@0: }
michael@0: 
michael@0: } // namespace WebCore