The Tor Browser: content/media/webspeech/recognition/energy

content/media/webspeech/recognition/energy_endpointer.cc@97036ab72558 (annotated)

content/media/webspeech/recognition/energy_endpointer.cc

Tue, 06 Jan 2015 21:39:09 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Tue, 06 Jan 2015 21:39:09 +0100
branch: TOR_BUG_9701
changeset 8: 97036ab72558
permissions: -rw-r--r--

Conditionally force memory storage according to privacy.thirdparty.isolate;
This solves Tor bug #9701, complying with disk avoidance documented in
https://www.torproject.org/projects/torbrowser/design/#disk-avoidance.

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //    * Redistributions of source code must retain the above copyright
 // notice, this list of conditions and the following disclaimer.
 //    * Redistributions in binary form must reproduce the above
 // copyright notice, this list of conditions and the following disclaimer
 // in the documentation and/or other materials provided with the
 // distribution.
 //    * Neither the name of Google Inc. nor the names of its
 // contributors may be used to endorse or promote products derived from
 // this software without specific prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "energy_endpointer.h"
 #include <math.h>
 namespace {
 // Returns the RMS (quadratic mean) of the input signal.
 float RMS(const int16_t* samples, int num_samples) {
   int64_t ssq_int64_t = 0;
   int64_t sum_int64_t = 0;
   for (int i = 0; i < num_samples; ++i) {
     sum_int64_t += samples[i];
     ssq_int64_t += samples[i] * samples[i];
   }
   // now convert to floats.
   double sum = static_cast<double>(sum_int64_t);
   sum /= num_samples;
   double ssq = static_cast<double>(ssq_int64_t);
   return static_cast<float>(sqrt((ssq / num_samples) - (sum * sum)));
 }
 int64_t Secs2Usecs(float seconds) {
   return static_cast<int64_t>(0.5 + (1.0e6 * seconds));
 }
 float GetDecibel(float value) {
   if (value > 1.0e-100)
     return 20 * log10(value);
   return -2000.0;
 }
 }  // namespace
 namespace mozilla {
 // Stores threshold-crossing histories for making decisions about the speech
 // state.
 class EnergyEndpointer::HistoryRing {
  public:
   HistoryRing() : insertion_index_(0) {}
   // Resets the ring to |size| elements each with state |initial_state|
   void SetRing(int size, bool initial_state);
   // Inserts a new entry into the ring and drops the oldest entry.
   void Insert(int64_t time_us, bool decision);
   // Returns the time in microseconds of the most recently added entry.
   int64_t EndTime() const;
   // Returns the sum of all intervals during which 'decision' is true within
   // the time in seconds specified by 'duration'. The returned interval is
   // in seconds.
   float RingSum(float duration_sec);
  private:
   struct DecisionPoint {
     int64_t time_us;
     bool decision;
   };
   std::vector<DecisionPoint> decision_points_;
   int insertion_index_;  // Index at which the next item gets added/inserted.
   HistoryRing(const HistoryRing&);
   void operator=(const HistoryRing&);
 };
 void EnergyEndpointer::HistoryRing::SetRing(int size, bool initial_state) {
   insertion_index_ = 0;
   decision_points_.clear();
   DecisionPoint init = { -1, initial_state };
   decision_points_.resize(size, init);
 }
 void EnergyEndpointer::HistoryRing::Insert(int64_t time_us, bool decision) {
   decision_points_[insertion_index_].time_us = time_us;
   decision_points_[insertion_index_].decision = decision;
   insertion_index_ = (insertion_index_ + 1) % decision_points_.size();
 }
 int64_t EnergyEndpointer::HistoryRing::EndTime() const {
   int ind = insertion_index_ - 1;
   if (ind < 0)
     ind = decision_points_.size() - 1;
   return decision_points_[ind].time_us;
 }
 float EnergyEndpointer::HistoryRing::RingSum(float duration_sec) {
   if (!decision_points_.size())
     return 0.0;
   int64_t sum_us = 0;
   int ind = insertion_index_ - 1;
   if (ind < 0)
     ind = decision_points_.size() - 1;
   int64_t end_us = decision_points_[ind].time_us;
   bool is_on = decision_points_[ind].decision;
   int64_t start_us = end_us - static_cast<int64_t>(0.5 + (1.0e6 * duration_sec));
   if (start_us < 0)
     start_us = 0;
   size_t n_summed = 1;  // n points ==> (n-1) intervals
   while ((decision_points_[ind].time_us > start_us) &&
          (n_summed < decision_points_.size())) {
     --ind;
     if (ind < 0)
       ind = decision_points_.size() - 1;
     if (is_on)
       sum_us += end_us - decision_points_[ind].time_us;
     is_on = decision_points_[ind].decision;
     end_us = decision_points_[ind].time_us;
     n_summed++;
   }
   return 1.0e-6f * sum_us;  //  Returns total time that was super threshold.
 }
 EnergyEndpointer::EnergyEndpointer()
     : status_(EP_PRE_SPEECH),
       offset_confirm_dur_sec_(0),
       endpointer_time_us_(0),
       fast_update_frames_(0),
       frame_counter_(0),
       max_window_dur_(4.0),
       sample_rate_(0),
       history_(new HistoryRing()),
       decision_threshold_(0),
       estimating_environment_(false),
       noise_level_(0),
       rms_adapt_(0),
       start_lag_(0),
       end_lag_(0),
       user_input_start_time_us_(0) {
 }
 EnergyEndpointer::~EnergyEndpointer() {
 }
 int EnergyEndpointer::TimeToFrame(float time) const {
   return static_cast<int32_t>(0.5 + (time / params_.frame_period()));
 }
 void EnergyEndpointer::Restart(bool reset_threshold) {
   status_ = EP_PRE_SPEECH;
   user_input_start_time_us_ = 0;
   if (reset_threshold) {
     decision_threshold_ = params_.decision_threshold();
     rms_adapt_ = decision_threshold_;
     noise_level_ = params_.decision_threshold() / 2.0f;
     frame_counter_ = 0;  // Used for rapid initial update of levels.
   }
   // Set up the memories to hold the history windows.
   history_->SetRing(TimeToFrame(max_window_dur_), false);
   // Flag that indicates that current input should be used for
   // estimating the environment. The user has not yet started input
   // by e.g. pressed the push-to-talk button. By default, this is
   // false for backward compatibility.
   estimating_environment_ = false;
 }
 void EnergyEndpointer::Init(const EnergyEndpointerParams& params) {
   params_ = params;
   // Find the longest history interval to be used, and make the ring
   // large enough to accommodate that number of frames.  NOTE: This
   // depends upon ep_frame_period being set correctly in the factory
   // that did this instantiation.
   max_window_dur_ = params_.onset_window();
   if (params_.speech_on_window() > max_window_dur_)
     max_window_dur_ = params_.speech_on_window();
   if (params_.offset_window() > max_window_dur_)
     max_window_dur_ = params_.offset_window();
   Restart(true);
   offset_confirm_dur_sec_ = params_.offset_window() -
                             params_.offset_confirm_dur();
   if (offset_confirm_dur_sec_ < 0.0)
     offset_confirm_dur_sec_ = 0.0;
   user_input_start_time_us_ = 0;
   // Flag that indicates that  current input should be used for
   // estimating the environment. The user has not yet started input
   // by e.g. pressed the push-to-talk button. By default, this is
   // false for backward compatibility.
   estimating_environment_ = false;
   // The initial value of the noise and speech levels is inconsequential.
   // The level of the first frame will overwrite these values.
   noise_level_ = params_.decision_threshold() / 2.0f;
   fast_update_frames_ =
       static_cast<int64_t>(params_.fast_update_dur() / params_.frame_period());
   frame_counter_ = 0;  // Used for rapid initial update of levels.
   sample_rate_ = params_.sample_rate();
   start_lag_ = static_cast<int>(sample_rate_ /
                                 params_.max_fundamental_frequency());
   end_lag_ = static_cast<int>(sample_rate_ /
                               params_.min_fundamental_frequency());
 }
 void EnergyEndpointer::StartSession() {
   Restart(true);
 }
 void EnergyEndpointer::EndSession() {
   status_ = EP_POST_SPEECH;
 }
 void EnergyEndpointer::SetEnvironmentEstimationMode() {
   Restart(true);
   estimating_environment_ = true;
 }
 void EnergyEndpointer::SetUserInputMode() {
   estimating_environment_ = false;
   user_input_start_time_us_ = endpointer_time_us_;
 }
 void EnergyEndpointer::ProcessAudioFrame(int64_t time_us,
                                          const int16_t* samples,
                                          int num_samples,
                                          float* rms_out) {
   endpointer_time_us_ = time_us;
   float rms = RMS(samples, num_samples);
   // Check that this is user input audio vs. pre-input adaptation audio.
   // Input audio starts when the user indicates start of input, by e.g.
   // pressing push-to-talk. Audio recieved prior to that is used to update
   // noise and speech level estimates.
   if (!estimating_environment_) {
     bool decision = false;
     if ((endpointer_time_us_ - user_input_start_time_us_) <
         Secs2Usecs(params_.contamination_rejection_period())) {
       decision = false;
       //PR_LOG(GetSpeechRecognitionLog(), PR_LOG_DEBUG, ("decision: forced to false, time: %d", endpointer_time_us_));
     } else {
       decision = (rms > decision_threshold_);
     }
     history_->Insert(endpointer_time_us_, decision);
     switch (status_) {
       case EP_PRE_SPEECH:
         if (history_->RingSum(params_.onset_window()) >
             params_.onset_detect_dur()) {
           status_ = EP_POSSIBLE_ONSET;
         }
         break;
       case EP_POSSIBLE_ONSET: {
         float tsum = history_->RingSum(params_.onset_window());
         if (tsum > params_.onset_confirm_dur()) {
           status_ = EP_SPEECH_PRESENT;
         } else {  // If signal is not maintained, drop back to pre-speech.
           if (tsum <= params_.onset_detect_dur())
             status_ = EP_PRE_SPEECH;
         }
         break;
       }
       case EP_SPEECH_PRESENT: {
         // To induce hysteresis in the state residency, we allow a
         // smaller residency time in the on_ring, than was required to
         // enter the SPEECH_PERSENT state.
         float on_time = history_->RingSum(params_.speech_on_window());
         if (on_time < params_.on_maintain_dur())
           status_ = EP_POSSIBLE_OFFSET;
         break;
       }
       case EP_POSSIBLE_OFFSET:
         if (history_->RingSum(params_.offset_window()) <=
             offset_confirm_dur_sec_) {
           // Note that this offset time may be beyond the end
           // of the input buffer in a real-time system.  It will be up
           // to the RecognizerSession to decide what to do.
           status_ = EP_PRE_SPEECH;  // Automatically reset for next utterance.
         } else {  // If speech picks up again we allow return to SPEECH_PRESENT.
           if (history_->RingSum(params_.speech_on_window()) >=
               params_.on_maintain_dur())
             status_ = EP_SPEECH_PRESENT;
         }
         break;
       default:
         break;
     }
     // If this is a quiet, non-speech region, slowly adapt the detection
     // threshold to be about 6dB above the average RMS.
     if ((!decision) && (status_ == EP_PRE_SPEECH)) {
       decision_threshold_ = (0.98f * decision_threshold_) + (0.02f * 2 * rms);
       rms_adapt_ = decision_threshold_;
     } else {
       // If this is in a speech region, adapt the decision threshold to
       // be about 10dB below the average RMS. If the noise level is high,
       // the threshold is pushed up.
       // Adaptation up to a higher level is 5 times faster than decay to
       // a lower level.
       if ((status_ == EP_SPEECH_PRESENT) && decision) {
         if (rms_adapt_ > rms) {
           rms_adapt_ = (0.99f * rms_adapt_) + (0.01f * rms);
         } else {
           rms_adapt_ = (0.95f * rms_adapt_) + (0.05f * rms);
         }
         float target_threshold = 0.3f * rms_adapt_ +  noise_level_;
         decision_threshold_ = (.90f * decision_threshold_) +
                               (0.10f * target_threshold);
       }
     }
     // Set a floor
     if (decision_threshold_ < params_.min_decision_threshold())
       decision_threshold_ = params_.min_decision_threshold();
   }
   // Update speech and noise levels.
   UpdateLevels(rms);
   ++frame_counter_;
   if (rms_out)
     *rms_out = GetDecibel(rms);
 }
 float EnergyEndpointer::GetNoiseLevelDb() const {
   return GetDecibel(noise_level_);
 }
 void EnergyEndpointer::UpdateLevels(float rms) {
   // Update quickly initially. We assume this is noise and that
   // speech is 6dB above the noise.
   if (frame_counter_ < fast_update_frames_) {
     // Alpha increases from 0 to (k-1)/k where k is the number of time
     // steps in the initial adaptation period.
     float alpha = static_cast<float>(frame_counter_) /
         static_cast<float>(fast_update_frames_);
     noise_level_ = (alpha * noise_level_) + ((1 - alpha) * rms);
     //PR_LOG(GetSpeechRecognitionLog(), PR_LOG_DEBUG, ("FAST UPDATE, frame_counter_ %d, fast_update_frames_ %d", frame_counter_, fast_update_frames_));
   } else {
     // Update Noise level. The noise level adapts quickly downward, but
     // slowly upward. The noise_level_ parameter is not currently used
     // for threshold adaptation. It is used for UI feedback.
     if (noise_level_ < rms)
       noise_level_ = (0.999f * noise_level_) + (0.001f * rms);
     else
       noise_level_ = (0.95f * noise_level_) + (0.05f * rms);
   }
   if (estimating_environment_ || (frame_counter_ < fast_update_frames_)) {
     decision_threshold_ = noise_level_ * 2; // 6dB above noise level.
     // Set a floor
     if (decision_threshold_ < params_.min_decision_threshold())
       decision_threshold_ = params_.min_decision_threshold();
   }
 }
 EpStatus EnergyEndpointer::Status(int64_t* status_time)  const {
   *status_time = history_->EndTime();
   return status_;
 }
 }  // namespace mozilla

The Tor Browser / annotate

content/media/webspeech/recognition/energy_endpointer.cc@97036ab72558 (annotated)

content/media/webspeech/recognition/energy_endpointer.cc