The Tor Browser / annotate

content/media/webspeech/recognition/SpeechRecognition.h@97036ab72558 (annotated)

content/media/webspeech/recognition/SpeechRecognition.h

Tue, 06 Jan 2015 21:39:09 +0100

author
Michael Schloh von Bennewitz <michael@schloh.com>
date
Tue, 06 Jan 2015 21:39:09 +0100
branch
TOR_BUG_9701
changeset 8
97036ab72558
permissions
-rw-r--r--

Conditionally force memory storage according to privacy.thirdparty.isolate;
This solves Tor bug #9701, complying with disk avoidance documented in
https://www.torproject.org/projects/torbrowser/design/#disk-avoidance.

michael@0 1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
michael@0 2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
michael@0 3 /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0 4 * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0 6
michael@0 7 #ifndef mozilla_dom_SpeechRecognition_h
michael@0 8 #define mozilla_dom_SpeechRecognition_h
michael@0 9
michael@0 10 #include "mozilla/Attributes.h"
michael@0 11 #include "mozilla/DOMEventTargetHelper.h"
michael@0 12 #include "nsCOMPtr.h"
michael@0 13 #include "nsString.h"
michael@0 14 #include "nsWrapperCache.h"
michael@0 15 #include "nsTArray.h"
michael@0 16 #include "js/TypeDecls.h"
michael@0 17
michael@0 18 #include "nsIDOMNavigatorUserMedia.h"
michael@0 19 #include "nsITimer.h"
michael@0 20 #include "MediaEngine.h"
michael@0 21 #include "MediaStreamGraph.h"
michael@0 22 #include "AudioSegment.h"
michael@0 23 #include "mozilla/WeakPtr.h"
michael@0 24 #include "mozilla/Preferences.h"
michael@0 25
michael@0 26 #include "SpeechGrammarList.h"
michael@0 27 #include "SpeechRecognitionResultList.h"
michael@0 28 #include "SpeechStreamListener.h"
michael@0 29 #include "nsISpeechRecognitionService.h"
michael@0 30 #include "endpointer.h"
michael@0 31
michael@0 32 #include "mozilla/dom/SpeechRecognitionError.h"
michael@0 33
michael@0 34 class nsIDOMWindow;
michael@0 35
michael@0 36 namespace mozilla {
michael@0 37
michael@0 38 namespace dom {
michael@0 39
michael@0 40 #define TEST_PREFERENCE_ENABLE "media.webspeech.test.enable"
michael@0 41 #define TEST_PREFERENCE_FAKE_FSM_EVENTS "media.webspeech.test.fake_fsm_events"
michael@0 42 #define TEST_PREFERENCE_FAKE_RECOGNITION_SERVICE "media.webspeech.test.fake_recognition_service"
michael@0 43 #define SPEECH_RECOGNITION_TEST_EVENT_REQUEST_TOPIC "SpeechRecognitionTest:RequestEvent"
michael@0 44 #define SPEECH_RECOGNITION_TEST_END_TOPIC "SpeechRecognitionTest:End"
michael@0 45
michael@0 46 class GlobalObject;
michael@0 47 class SpeechEvent;
michael@0 48
michael@0 49 #ifdef PR_LOGGING
michael@0 50 PRLogModuleInfo* GetSpeechRecognitionLog();
michael@0 51 #define SR_LOG(...) PR_LOG(GetSpeechRecognitionLog(), PR_LOG_DEBUG, (__VA_ARGS__))
michael@0 52 #else
michael@0 53 #define SR_LOG(...)
michael@0 54 #endif
michael@0 55
michael@0 56 class SpeechRecognition MOZ_FINAL : public DOMEventTargetHelper,
michael@0 57 public nsIObserver,
michael@0 58 public SupportsWeakPtr<SpeechRecognition>
michael@0 59 {
michael@0 60 public:
michael@0 61 MOZ_DECLARE_REFCOUNTED_TYPENAME(SpeechRecognition)
michael@0 62 SpeechRecognition(nsPIDOMWindow* aOwnerWindow);
michael@0 63 virtual ~SpeechRecognition() {};
michael@0 64
michael@0 65 NS_DECL_ISUPPORTS_INHERITED
michael@0 66
michael@0 67 NS_DECL_NSIOBSERVER
michael@0 68
michael@0 69 nsISupports* GetParentObject() const;
michael@0 70
michael@0 71 virtual JSObject* WrapObject(JSContext* aCx) MOZ_OVERRIDE;
michael@0 72
michael@0 73 static already_AddRefed<SpeechRecognition>
michael@0 74 Constructor(const GlobalObject& aGlobal, ErrorResult& aRv);
michael@0 75
michael@0 76 already_AddRefed<SpeechGrammarList> GetGrammars(ErrorResult& aRv) const;
michael@0 77
michael@0 78 void SetGrammars(mozilla::dom::SpeechGrammarList& aArg, ErrorResult& aRv);
michael@0 79
michael@0 80 void GetLang(nsString& aRetVal, ErrorResult& aRv) const;
michael@0 81
michael@0 82 void SetLang(const nsAString& aArg, ErrorResult& aRv);
michael@0 83
michael@0 84 bool GetContinuous(ErrorResult& aRv) const;
michael@0 85
michael@0 86 void SetContinuous(bool aArg, ErrorResult& aRv);
michael@0 87
michael@0 88 bool GetInterimResults(ErrorResult& aRv) const;
michael@0 89
michael@0 90 void SetInterimResults(bool aArg, ErrorResult& aRv);
michael@0 91
michael@0 92 uint32_t GetMaxAlternatives(ErrorResult& aRv) const;
michael@0 93
michael@0 94 void SetMaxAlternatives(uint32_t aArg, ErrorResult& aRv);
michael@0 95
michael@0 96 void GetServiceURI(nsString& aRetVal, ErrorResult& aRv) const;
michael@0 97
michael@0 98 void SetServiceURI(const nsAString& aArg, ErrorResult& aRv);
michael@0 99
michael@0 100 void Start(ErrorResult& aRv);
michael@0 101
michael@0 102 void Stop();
michael@0 103
michael@0 104 void Abort();
michael@0 105
michael@0 106 IMPL_EVENT_HANDLER(audiostart)
michael@0 107 IMPL_EVENT_HANDLER(soundstart)
michael@0 108 IMPL_EVENT_HANDLER(speechstart)
michael@0 109 IMPL_EVENT_HANDLER(speechend)
michael@0 110 IMPL_EVENT_HANDLER(soundend)
michael@0 111 IMPL_EVENT_HANDLER(audioend)
michael@0 112 IMPL_EVENT_HANDLER(result)
michael@0 113 IMPL_EVENT_HANDLER(nomatch)
michael@0 114 IMPL_EVENT_HANDLER(error)
michael@0 115 IMPL_EVENT_HANDLER(start)
michael@0 116 IMPL_EVENT_HANDLER(end)
michael@0 117
michael@0 118 enum EventType {
michael@0 119 EVENT_START,
michael@0 120 EVENT_STOP,
michael@0 121 EVENT_ABORT,
michael@0 122 EVENT_AUDIO_DATA,
michael@0 123 EVENT_AUDIO_ERROR,
michael@0 124 EVENT_RECOGNITIONSERVICE_INTERMEDIATE_RESULT,
michael@0 125 EVENT_RECOGNITIONSERVICE_FINAL_RESULT,
michael@0 126 EVENT_RECOGNITIONSERVICE_ERROR,
michael@0 127 EVENT_COUNT
michael@0 128 };
michael@0 129
michael@0 130 void DispatchError(EventType aErrorType, SpeechRecognitionErrorCode aErrorCode, const nsAString& aMessage);
michael@0 131 uint32_t FillSamplesBuffer(const int16_t* aSamples, uint32_t aSampleCount);
michael@0 132 uint32_t SplitSamplesBuffer(const int16_t* aSamplesBuffer, uint32_t aSampleCount, nsTArray<nsRefPtr<SharedBuffer>>& aResult);
michael@0 133 AudioSegment* CreateAudioSegment(nsTArray<nsRefPtr<SharedBuffer>>& aChunks);
michael@0 134 void FeedAudioData(already_AddRefed<SharedBuffer> aSamples, uint32_t aDuration, MediaStreamListener* aProvider);
michael@0 135
michael@0 136 static struct TestConfig
michael@0 137 {
michael@0 138 public:
michael@0 139 bool mEnableTests;
michael@0 140 bool mFakeFSMEvents;
michael@0 141 bool mFakeRecognitionService;
michael@0 142
michael@0 143 void Init()
michael@0 144 {
michael@0 145 if (mInitialized) {
michael@0 146 return;
michael@0 147 }
michael@0 148
michael@0 149 Preferences::AddBoolVarCache(&mEnableTests, TEST_PREFERENCE_ENABLE);
michael@0 150
michael@0 151 if (mEnableTests) {
michael@0 152 Preferences::AddBoolVarCache(&mFakeFSMEvents, TEST_PREFERENCE_FAKE_FSM_EVENTS);
michael@0 153 Preferences::AddBoolVarCache(&mFakeRecognitionService, TEST_PREFERENCE_FAKE_RECOGNITION_SERVICE);
michael@0 154 }
michael@0 155
michael@0 156 mInitialized = true;
michael@0 157 }
michael@0 158 private:
michael@0 159 bool mInitialized;
michael@0 160 } mTestConfig;
michael@0 161
michael@0 162
michael@0 163 friend class SpeechEvent;
michael@0 164 private:
michael@0 165 enum FSMState {
michael@0 166 STATE_IDLE,
michael@0 167 STATE_STARTING,
michael@0 168 STATE_ESTIMATING,
michael@0 169 STATE_WAITING_FOR_SPEECH,
michael@0 170 STATE_RECOGNIZING,
michael@0 171 STATE_WAITING_FOR_RESULT,
michael@0 172 STATE_COUNT
michael@0 173 };
michael@0 174
michael@0 175 void SetState(FSMState state);
michael@0 176 bool StateBetween(FSMState begin, FSMState end);
michael@0 177
michael@0 178 class GetUserMediaSuccessCallback : public nsIDOMGetUserMediaSuccessCallback
michael@0 179 {
michael@0 180 public:
michael@0 181 NS_DECL_ISUPPORTS
michael@0 182 NS_DECL_NSIDOMGETUSERMEDIASUCCESSCALLBACK
michael@0 183
michael@0 184 GetUserMediaSuccessCallback(SpeechRecognition* aRecognition)
michael@0 185 : mRecognition(aRecognition)
michael@0 186 {}
michael@0 187
michael@0 188 virtual ~GetUserMediaSuccessCallback() {}
michael@0 189
michael@0 190 private:
michael@0 191 nsRefPtr<SpeechRecognition> mRecognition;
michael@0 192 };
michael@0 193
michael@0 194 class GetUserMediaErrorCallback : public nsIDOMGetUserMediaErrorCallback
michael@0 195 {
michael@0 196 public:
michael@0 197 NS_DECL_ISUPPORTS
michael@0 198 NS_DECL_NSIDOMGETUSERMEDIAERRORCALLBACK
michael@0 199
michael@0 200 GetUserMediaErrorCallback(SpeechRecognition* aRecognition)
michael@0 201 : mRecognition(aRecognition)
michael@0 202 {}
michael@0 203
michael@0 204 virtual ~GetUserMediaErrorCallback() {}
michael@0 205
michael@0 206 private:
michael@0 207 nsRefPtr<SpeechRecognition> mRecognition;
michael@0 208 };
michael@0 209
michael@0 210 NS_IMETHOD StartRecording(DOMMediaStream* aDOMStream);
michael@0 211 NS_IMETHOD StopRecording();
michael@0 212
michael@0 213 uint32_t ProcessAudioSegment(AudioSegment* aSegment);
michael@0 214 void NotifyError(SpeechEvent* aEvent);
michael@0 215
michael@0 216 void ProcessEvent(SpeechEvent* aEvent);
michael@0 217 void Transition(SpeechEvent* aEvent);
michael@0 218
michael@0 219 void Reset();
michael@0 220 void ResetAndEnd();
michael@0 221 void WaitForAudioData(SpeechEvent* aEvent);
michael@0 222 void StartedAudioCapture(SpeechEvent* aEvent);
michael@0 223 void StopRecordingAndRecognize(SpeechEvent* aEvent);
michael@0 224 void WaitForEstimation(SpeechEvent* aEvent);
michael@0 225 void DetectSpeech(SpeechEvent* aEvent);
michael@0 226 void WaitForSpeechEnd(SpeechEvent* aEvent);
michael@0 227 void NotifyFinalResult(SpeechEvent* aEvent);
michael@0 228 void DoNothing(SpeechEvent* aEvent);
michael@0 229 void AbortSilently(SpeechEvent* aEvent);
michael@0 230 void AbortError(SpeechEvent* aEvent);
michael@0 231
michael@0 232 nsRefPtr<DOMMediaStream> mDOMStream;
michael@0 233 nsRefPtr<SpeechStreamListener> mSpeechListener;
michael@0 234 nsCOMPtr<nsISpeechRecognitionService> mRecognitionService;
michael@0 235
michael@0 236 void GetRecognitionServiceCID(nsACString& aResultCID);
michael@0 237
michael@0 238 FSMState mCurrentState;
michael@0 239
michael@0 240 Endpointer mEndpointer;
michael@0 241 uint32_t mEstimationSamples;
michael@0 242
michael@0 243 uint32_t mAudioSamplesPerChunk;
michael@0 244
michael@0 245 // buffer holds one chunk of mAudioSamplesPerChunk
michael@0 246 // samples before feeding it to mEndpointer
michael@0 247 nsRefPtr<SharedBuffer> mAudioSamplesBuffer;
michael@0 248 uint32_t mBufferedSamples;
michael@0 249
michael@0 250 nsCOMPtr<nsITimer> mSpeechDetectionTimer;
michael@0 251 bool mAborted;
michael@0 252
michael@0 253 void ProcessTestEventRequest(nsISupports* aSubject, const nsAString& aEventName);
michael@0 254
michael@0 255 const char* GetName(FSMState aId);
michael@0 256 const char* GetName(SpeechEvent* aId);
michael@0 257 };
michael@0 258
michael@0 259 class SpeechEvent : public nsRunnable
michael@0 260 {
michael@0 261 public:
michael@0 262 SpeechEvent(SpeechRecognition* aRecognition, SpeechRecognition::EventType aType)
michael@0 263 : mAudioSegment(0)
michael@0 264 , mRecognitionResultList(0)
michael@0 265 , mError(0)
michael@0 266 , mRecognition(aRecognition)
michael@0 267 , mType(aType)
michael@0 268 {
michael@0 269 }
michael@0 270
michael@0 271 ~SpeechEvent();
michael@0 272
michael@0 273 NS_IMETHOD Run() MOZ_OVERRIDE;
michael@0 274 AudioSegment* mAudioSegment;
michael@0 275 nsRefPtr<SpeechRecognitionResultList> mRecognitionResultList; // TODO: make this a session being passed which also has index and stuff
michael@0 276 nsRefPtr<SpeechRecognitionError> mError;
michael@0 277
michael@0 278 friend class SpeechRecognition;
michael@0 279 private:
michael@0 280 SpeechRecognition* mRecognition;
michael@0 281
michael@0 282 // for AUDIO_DATA events, keep a reference to the provider
michael@0 283 // of the data (i.e., the SpeechStreamListener) to ensure it
michael@0 284 // is kept alive (and keeps SpeechRecognition alive) until this
michael@0 285 // event gets processed.
michael@0 286 nsRefPtr<MediaStreamListener> mProvider;
michael@0 287 SpeechRecognition::EventType mType;
michael@0 288 };
michael@0 289
michael@0 290 } // namespace dom
michael@0 291
michael@0 292 inline nsISupports*
michael@0 293 ToSupports(dom::SpeechRecognition* aRec)
michael@0 294 {
michael@0 295 return ToSupports(static_cast<DOMEventTargetHelper*>(aRec));
michael@0 296 }
michael@0 297 } // namespace mozilla
michael@0 298
michael@0 299 #endif

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial