|
1 /* This Source Code Form is subject to the terms of the Mozilla Public |
|
2 * License, v. 2.0. If a copy of the MPL was not distributed with this |
|
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ |
|
4 |
|
5 "use strict"; |
|
6 |
|
7 const {classes: Cc, interfaces: Ci, utils: Cu} = Components; |
|
8 |
|
9 Cu.import("resource://gre/modules/Log.jsm", this); |
|
10 Cu.import("resource://gre/modules/osfile.jsm", this) |
|
11 Cu.import("resource://gre/modules/Promise.jsm", this); |
|
12 Cu.import("resource://gre/modules/Services.jsm", this); |
|
13 Cu.import("resource://gre/modules/Task.jsm", this); |
|
14 Cu.import("resource://gre/modules/Timer.jsm", this); |
|
15 Cu.import("resource://gre/modules/XPCOMUtils.jsm", this); |
|
16 Cu.import("resource://services-common/utils.js", this); |
|
17 |
|
18 this.EXPORTED_SYMBOLS = [ |
|
19 "CrashManager", |
|
20 ]; |
|
21 |
|
22 /** |
|
23 * How long to wait after application startup before crash event files are |
|
24 * automatically aggregated. |
|
25 * |
|
26 * We defer aggregation for performance reasons, as we don't want too many |
|
27 * services competing for I/O immediately after startup. |
|
28 */ |
|
29 const AGGREGATE_STARTUP_DELAY_MS = 57000; |
|
30 |
|
31 const MILLISECONDS_IN_DAY = 24 * 60 * 60 * 1000; |
|
32 |
|
33 // Converts Date to days since UNIX epoch. |
|
34 // This was copied from /services/metrics.storage.jsm. The implementation |
|
35 // does not account for leap seconds. |
|
36 function dateToDays(date) { |
|
37 return Math.floor(date.getTime() / MILLISECONDS_IN_DAY); |
|
38 } |
|
39 |
|
40 |
|
41 /** |
|
42 * A gateway to crash-related data. |
|
43 * |
|
44 * This type is generic and can be instantiated any number of times. |
|
45 * However, most applications will typically only have one instance |
|
46 * instantiated and that instance will point to profile and user appdata |
|
47 * directories. |
|
48 * |
|
49 * Instances are created by passing an object with properties. |
|
50 * Recognized properties are: |
|
51 * |
|
52 * pendingDumpsDir (string) (required) |
|
53 * Where dump files that haven't been uploaded are located. |
|
54 * |
|
55 * submittedDumpsDir (string) (required) |
|
56 * Where records of uploaded dumps are located. |
|
57 * |
|
58 * eventsDirs (array) |
|
59 * Directories (defined as strings) where events files are written. This |
|
60 * instance will collects events from files in the directories specified. |
|
61 * |
|
62 * storeDir (string) |
|
63 * Directory we will use for our data store. This instance will write |
|
64 * data files into the directory specified. |
|
65 * |
|
66 * telemetryStoreSizeKey (string) |
|
67 * Telemetry histogram to report store size under. |
|
68 */ |
|
69 this.CrashManager = function (options) { |
|
70 for (let k of ["pendingDumpsDir", "submittedDumpsDir", "eventsDirs", |
|
71 "storeDir"]) { |
|
72 if (!(k in options)) { |
|
73 throw new Error("Required key not present in options: " + k); |
|
74 } |
|
75 } |
|
76 |
|
77 this._log = Log.repository.getLogger("Crashes.CrashManager"); |
|
78 |
|
79 for (let k in options) { |
|
80 let v = options[k]; |
|
81 |
|
82 switch (k) { |
|
83 case "pendingDumpsDir": |
|
84 this._pendingDumpsDir = v; |
|
85 break; |
|
86 |
|
87 case "submittedDumpsDir": |
|
88 this._submittedDumpsDir = v; |
|
89 break; |
|
90 |
|
91 case "eventsDirs": |
|
92 this._eventsDirs = v; |
|
93 break; |
|
94 |
|
95 case "storeDir": |
|
96 this._storeDir = v; |
|
97 break; |
|
98 |
|
99 case "telemetryStoreSizeKey": |
|
100 this._telemetryStoreSizeKey = v; |
|
101 break; |
|
102 |
|
103 default: |
|
104 throw new Error("Unknown property in options: " + k); |
|
105 } |
|
106 } |
|
107 |
|
108 // Promise for in-progress aggregation operation. We store it on the |
|
109 // object so it can be returned for in-progress operations. |
|
110 this._aggregatePromise = null; |
|
111 |
|
112 // The CrashStore currently attached to this object. |
|
113 this._store = null; |
|
114 |
|
115 // The timer controlling the expiration of the CrashStore instance. |
|
116 this._storeTimer = null; |
|
117 |
|
118 // This is a semaphore that prevents the store from being freed by our |
|
119 // timer-based resource freeing mechanism. |
|
120 this._storeProtectedCount = 0; |
|
121 }; |
|
122 |
|
123 this.CrashManager.prototype = Object.freeze({ |
|
124 DUMP_REGEX: /^([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.dmp$/i, |
|
125 SUBMITTED_REGEX: /^bp-(?:hr-)?([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.txt$/i, |
|
126 ALL_REGEX: /^(.*)$/, |
|
127 |
|
128 // How long the store object should persist in memory before being |
|
129 // automatically garbage collected. |
|
130 STORE_EXPIRATION_MS: 60 * 1000, |
|
131 |
|
132 // Number of days after which a crash with no activity will get purged. |
|
133 PURGE_OLDER_THAN_DAYS: 180, |
|
134 |
|
135 // The following are return codes for individual event file processing. |
|
136 // File processed OK. |
|
137 EVENT_FILE_SUCCESS: "ok", |
|
138 // The event appears to be malformed. |
|
139 EVENT_FILE_ERROR_MALFORMED: "malformed", |
|
140 // The type of event is unknown. |
|
141 EVENT_FILE_ERROR_UNKNOWN_EVENT: "unknown-event", |
|
142 |
|
143 /** |
|
144 * Obtain a list of all dumps pending upload. |
|
145 * |
|
146 * The returned value is a promise that resolves to an array of objects |
|
147 * on success. Each element in the array has the following properties: |
|
148 * |
|
149 * id (string) |
|
150 * The ID of the crash (a UUID). |
|
151 * |
|
152 * path (string) |
|
153 * The filename of the crash (<UUID.dmp>) |
|
154 * |
|
155 * date (Date) |
|
156 * When this dump was created |
|
157 * |
|
158 * The returned arry is sorted by the modified time of the file backing |
|
159 * the entry, oldest to newest. |
|
160 * |
|
161 * @return Promise<Array> |
|
162 */ |
|
163 pendingDumps: function () { |
|
164 return this._getDirectoryEntries(this._pendingDumpsDir, this.DUMP_REGEX); |
|
165 }, |
|
166 |
|
167 /** |
|
168 * Obtain a list of all dump files corresponding to submitted crashes. |
|
169 * |
|
170 * The returned value is a promise that resolves to an Array of |
|
171 * objects. Each object has the following properties: |
|
172 * |
|
173 * path (string) |
|
174 * The path of the file this entry comes from. |
|
175 * |
|
176 * id (string) |
|
177 * The crash UUID. |
|
178 * |
|
179 * date (Date) |
|
180 * The (estimated) date this crash was submitted. |
|
181 * |
|
182 * The returned array is sorted by the modified time of the file backing |
|
183 * the entry, oldest to newest. |
|
184 * |
|
185 * @return Promise<Array> |
|
186 */ |
|
187 submittedDumps: function () { |
|
188 return this._getDirectoryEntries(this._submittedDumpsDir, |
|
189 this.SUBMITTED_REGEX); |
|
190 }, |
|
191 |
|
192 /** |
|
193 * Aggregates "loose" events files into the unified "database." |
|
194 * |
|
195 * This function should be called periodically to collect metadata from |
|
196 * all events files into the central data store maintained by this manager. |
|
197 * |
|
198 * Once events have been stored in the backing store the corresponding |
|
199 * source files are deleted. |
|
200 * |
|
201 * Only one aggregation operation is allowed to occur at a time. If this |
|
202 * is called when an existing aggregation is in progress, the promise for |
|
203 * the original call will be returned. |
|
204 * |
|
205 * @return promise<int> The number of event files that were examined. |
|
206 */ |
|
207 aggregateEventsFiles: function () { |
|
208 if (this._aggregatePromise) { |
|
209 return this._aggregatePromise; |
|
210 } |
|
211 |
|
212 return this._aggregatePromise = Task.spawn(function* () { |
|
213 if (this._aggregatePromise) { |
|
214 return this._aggregatePromise; |
|
215 } |
|
216 |
|
217 try { |
|
218 let unprocessedFiles = yield this._getUnprocessedEventsFiles(); |
|
219 |
|
220 let deletePaths = []; |
|
221 let needsSave = false; |
|
222 |
|
223 this._storeProtectedCount++; |
|
224 for (let entry of unprocessedFiles) { |
|
225 try { |
|
226 let result = yield this._processEventFile(entry); |
|
227 |
|
228 switch (result) { |
|
229 case this.EVENT_FILE_SUCCESS: |
|
230 needsSave = true; |
|
231 // Fall through. |
|
232 |
|
233 case this.EVENT_FILE_ERROR_MALFORMED: |
|
234 deletePaths.push(entry.path); |
|
235 break; |
|
236 |
|
237 case this.EVENT_FILE_ERROR_UNKNOWN_EVENT: |
|
238 break; |
|
239 |
|
240 default: |
|
241 Cu.reportError("Unhandled crash event file return code. Please " + |
|
242 "file a bug: " + result); |
|
243 } |
|
244 } catch (ex if ex instanceof OS.File.Error) { |
|
245 this._log.warn("I/O error reading " + entry.path + ": " + |
|
246 CommonUtils.exceptionStr(ex)); |
|
247 } catch (ex) { |
|
248 // We should never encounter an exception. This likely represents |
|
249 // a coding error because all errors should be detected and |
|
250 // converted to return codes. |
|
251 // |
|
252 // If we get here, report the error and delete the source file |
|
253 // so we don't see it again. |
|
254 Cu.reportError("Exception when processing crash event file: " + |
|
255 CommonUtils.exceptionStr(ex)); |
|
256 deletePaths.push(entry.path); |
|
257 } |
|
258 } |
|
259 |
|
260 if (needsSave) { |
|
261 let store = yield this._getStore(); |
|
262 yield store.save(); |
|
263 } |
|
264 |
|
265 for (let path of deletePaths) { |
|
266 try { |
|
267 yield OS.File.remove(path); |
|
268 } catch (ex) { |
|
269 this._log.warn("Error removing event file (" + path + "): " + |
|
270 CommonUtils.exceptionStr(ex)); |
|
271 } |
|
272 } |
|
273 |
|
274 return unprocessedFiles.length; |
|
275 |
|
276 } finally { |
|
277 this._aggregatePromise = false; |
|
278 this._storeProtectedCount--; |
|
279 } |
|
280 }.bind(this)); |
|
281 }, |
|
282 |
|
283 /** |
|
284 * Prune old crash data. |
|
285 * |
|
286 * @param date |
|
287 * (Date) The cutoff point for pruning. Crashes without data newer |
|
288 * than this will be pruned. |
|
289 */ |
|
290 pruneOldCrashes: function (date) { |
|
291 return Task.spawn(function* () { |
|
292 let store = yield this._getStore(); |
|
293 store.pruneOldCrashes(date); |
|
294 yield store.save(); |
|
295 }.bind(this)); |
|
296 }, |
|
297 |
|
298 /** |
|
299 * Run tasks that should be periodically performed. |
|
300 */ |
|
301 runMaintenanceTasks: function () { |
|
302 return Task.spawn(function* () { |
|
303 yield this.aggregateEventsFiles(); |
|
304 |
|
305 let offset = this.PURGE_OLDER_THAN_DAYS * MILLISECONDS_IN_DAY; |
|
306 yield this.pruneOldCrashes(new Date(Date.now() - offset)); |
|
307 }.bind(this)); |
|
308 }, |
|
309 |
|
310 /** |
|
311 * Schedule maintenance tasks for some point in the future. |
|
312 * |
|
313 * @param delay |
|
314 * (integer) Delay in milliseconds when maintenance should occur. |
|
315 */ |
|
316 scheduleMaintenance: function (delay) { |
|
317 let deferred = Promise.defer(); |
|
318 |
|
319 setTimeout(() => { |
|
320 this.runMaintenanceTasks().then(deferred.resolve, deferred.reject); |
|
321 }, delay); |
|
322 |
|
323 return deferred.promise; |
|
324 }, |
|
325 |
|
326 /** |
|
327 * Obtain the paths of all unprocessed events files. |
|
328 * |
|
329 * The promise-resolved array is sorted by file mtime, oldest to newest. |
|
330 */ |
|
331 _getUnprocessedEventsFiles: function () { |
|
332 return Task.spawn(function* () { |
|
333 let entries = []; |
|
334 |
|
335 for (let dir of this._eventsDirs) { |
|
336 for (let e of yield this._getDirectoryEntries(dir, this.ALL_REGEX)) { |
|
337 entries.push(e); |
|
338 } |
|
339 } |
|
340 |
|
341 entries.sort((a, b) => { return a.date - b.date; }); |
|
342 |
|
343 return entries; |
|
344 }.bind(this)); |
|
345 }, |
|
346 |
|
347 // See docs/crash-events.rst for the file format specification. |
|
348 _processEventFile: function (entry) { |
|
349 return Task.spawn(function* () { |
|
350 let data = yield OS.File.read(entry.path); |
|
351 let store = yield this._getStore(); |
|
352 |
|
353 let decoder = new TextDecoder(); |
|
354 data = decoder.decode(data); |
|
355 |
|
356 let type, time, payload; |
|
357 let start = 0; |
|
358 for (let i = 0; i < 2; i++) { |
|
359 let index = data.indexOf("\n", start); |
|
360 if (index == -1) { |
|
361 return this.EVENT_FILE_ERROR_MALFORMED; |
|
362 } |
|
363 |
|
364 let sub = data.substring(start, index); |
|
365 switch (i) { |
|
366 case 0: |
|
367 type = sub; |
|
368 break; |
|
369 case 1: |
|
370 time = sub; |
|
371 try { |
|
372 time = parseInt(time, 10); |
|
373 } catch (ex) { |
|
374 return this.EVENT_FILE_ERROR_MALFORMED; |
|
375 } |
|
376 } |
|
377 |
|
378 start = index + 1; |
|
379 } |
|
380 let date = new Date(time * 1000); |
|
381 let payload = data.substring(start); |
|
382 |
|
383 return this._handleEventFilePayload(store, entry, type, date, payload); |
|
384 }.bind(this)); |
|
385 }, |
|
386 |
|
387 _handleEventFilePayload: function (store, entry, type, date, payload) { |
|
388 // The payload types and formats are documented in docs/crash-events.rst. |
|
389 // Do not change the format of an existing type. Instead, invent a new |
|
390 // type. |
|
391 |
|
392 let eventMap = { |
|
393 "crash.main.1": "addMainProcessCrash", |
|
394 "crash.plugin.1": "addPluginCrash", |
|
395 "hang.plugin.1": "addPluginHang", |
|
396 }; |
|
397 |
|
398 if (type in eventMap) { |
|
399 let lines = payload.split("\n"); |
|
400 if (lines.length > 1) { |
|
401 this._log.warn("Multiple lines unexpected in payload for " + |
|
402 entry.path); |
|
403 return this.EVENT_FILE_ERROR_MALFORMED; |
|
404 } |
|
405 |
|
406 store[eventMap[type]](payload, date); |
|
407 return this.EVENT_FILE_SUCCESS; |
|
408 } |
|
409 |
|
410 // DO NOT ADD NEW TYPES WITHOUT DOCUMENTING! |
|
411 |
|
412 return this.EVENT_FILE_ERROR_UNKNOWN_EVENT; |
|
413 }, |
|
414 |
|
415 /** |
|
416 * The resolved promise is an array of objects with the properties: |
|
417 * |
|
418 * path -- String filename |
|
419 * id -- regexp.match()[1] (likely the crash ID) |
|
420 * date -- Date mtime of the file |
|
421 */ |
|
422 _getDirectoryEntries: function (path, re) { |
|
423 return Task.spawn(function* () { |
|
424 try { |
|
425 yield OS.File.stat(path); |
|
426 } catch (ex if ex instanceof OS.File.Error && ex.becauseNoSuchFile) { |
|
427 return []; |
|
428 } |
|
429 |
|
430 let it = new OS.File.DirectoryIterator(path); |
|
431 let entries = []; |
|
432 |
|
433 try { |
|
434 yield it.forEach((entry, index, it) => { |
|
435 if (entry.isDir) { |
|
436 return; |
|
437 } |
|
438 |
|
439 let match = re.exec(entry.name); |
|
440 if (!match) { |
|
441 return; |
|
442 } |
|
443 |
|
444 return OS.File.stat(entry.path).then((info) => { |
|
445 entries.push({ |
|
446 path: entry.path, |
|
447 id: match[1], |
|
448 date: info.lastModificationDate, |
|
449 }); |
|
450 }); |
|
451 }); |
|
452 } finally { |
|
453 it.close(); |
|
454 } |
|
455 |
|
456 entries.sort((a, b) => { return a.date - b.date; }); |
|
457 |
|
458 return entries; |
|
459 }.bind(this)); |
|
460 }, |
|
461 |
|
462 _getStore: function () { |
|
463 return Task.spawn(function* () { |
|
464 if (!this._store) { |
|
465 yield OS.File.makeDir(this._storeDir, { |
|
466 ignoreExisting: true, |
|
467 unixMode: OS.Constants.libc.S_IRWXU, |
|
468 }); |
|
469 |
|
470 let store = new CrashStore(this._storeDir, this._telemetryStoreSizeKey); |
|
471 yield store.load(); |
|
472 |
|
473 this._store = store; |
|
474 this._storeTimer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer); |
|
475 } |
|
476 |
|
477 // The application can go long periods without interacting with the |
|
478 // store. Since the store takes up resources, we automatically "free" |
|
479 // the store after inactivity so resources can be returned to the system. |
|
480 // We do this via a timer and a mechanism that tracks when the store |
|
481 // is being accessed. |
|
482 this._storeTimer.cancel(); |
|
483 |
|
484 // This callback frees resources from the store unless the store |
|
485 // is protected from freeing by some other process. |
|
486 let timerCB = function () { |
|
487 if (this._storeProtectedCount) { |
|
488 this._storeTimer.initWithCallback(timerCB, this.STORE_EXPIRATION_MS, |
|
489 this._storeTimer.TYPE_ONE_SHOT); |
|
490 return; |
|
491 } |
|
492 |
|
493 // We kill the reference that we hold. GC will kill it later. If |
|
494 // someone else holds a reference, that will prevent GC until that |
|
495 // reference is gone. |
|
496 this._store = null; |
|
497 this._storeTimer = null; |
|
498 }.bind(this); |
|
499 |
|
500 this._storeTimer.initWithCallback(timerCB, this.STORE_EXPIRATION_MS, |
|
501 this._storeTimer.TYPE_ONE_SHOT); |
|
502 |
|
503 return this._store; |
|
504 }.bind(this)); |
|
505 }, |
|
506 |
|
507 /** |
|
508 * Obtain information about all known crashes. |
|
509 * |
|
510 * Returns an array of CrashRecord instances. Instances are read-only. |
|
511 */ |
|
512 getCrashes: function () { |
|
513 return Task.spawn(function* () { |
|
514 let store = yield this._getStore(); |
|
515 |
|
516 return store.crashes; |
|
517 }.bind(this)); |
|
518 }, |
|
519 |
|
520 getCrashCountsByDay: function () { |
|
521 return Task.spawn(function* () { |
|
522 let store = yield this._getStore(); |
|
523 |
|
524 return store._countsByDay; |
|
525 }.bind(this)); |
|
526 }, |
|
527 }); |
|
528 |
|
529 let gCrashManager; |
|
530 |
|
531 /** |
|
532 * Interface to storage of crash data. |
|
533 * |
|
534 * This type handles storage of crash metadata. It exists as a separate type |
|
535 * from the crash manager for performance reasons: since all crash metadata |
|
536 * needs to be loaded into memory for access, we wish to easily dispose of all |
|
537 * associated memory when this data is no longer needed. Having an isolated |
|
538 * object whose references can easily be lost faciliates that simple disposal. |
|
539 * |
|
540 * When metadata is updated, the caller must explicitly persist the changes |
|
541 * to disk. This prevents excessive I/O during updates. |
|
542 * |
|
543 * The store has a mechanism for ensuring it doesn't grow too large. A ceiling |
|
544 * is placed on the number of daily events that can occur for events that can |
|
545 * occur with relatively high frequency, notably plugin crashes and hangs |
|
546 * (plugins can enter cycles where they repeatedly crash). If we've reached |
|
547 * the high water mark and new data arrives, it's silently dropped. |
|
548 * However, the count of actual events is always preserved. This allows |
|
549 * us to report on the severity of problems beyond the storage threshold. |
|
550 * |
|
551 * Main process crashes are excluded from limits because they are both |
|
552 * important and should be rare. |
|
553 * |
|
554 * @param storeDir (string) |
|
555 * Directory the store should be located in. |
|
556 * @param telemetrySizeKey (string) |
|
557 * The telemetry histogram that should be used to store the size |
|
558 * of the data file. |
|
559 */ |
|
560 function CrashStore(storeDir, telemetrySizeKey) { |
|
561 this._storeDir = storeDir; |
|
562 this._telemetrySizeKey = telemetrySizeKey; |
|
563 |
|
564 this._storePath = OS.Path.join(storeDir, "store.json.mozlz4"); |
|
565 |
|
566 // Holds the read data from disk. |
|
567 this._data = null; |
|
568 |
|
569 // Maps days since UNIX epoch to a Map of event types to counts. |
|
570 // This data structure is populated when the JSON file is loaded |
|
571 // and is also updated when new events are added. |
|
572 this._countsByDay = new Map(); |
|
573 } |
|
574 |
|
575 CrashStore.prototype = Object.freeze({ |
|
576 // A crash that occurred in the main process. |
|
577 TYPE_MAIN_CRASH: "main-crash", |
|
578 |
|
579 // A crash in a plugin process. |
|
580 TYPE_PLUGIN_CRASH: "plugin-crash", |
|
581 |
|
582 // A hang in a plugin process. |
|
583 TYPE_PLUGIN_HANG: "plugin-hang", |
|
584 |
|
585 // Maximum number of events to store per day. This establishes a |
|
586 // ceiling on the per-type/per-day records that will be stored. |
|
587 HIGH_WATER_DAILY_THRESHOLD: 100, |
|
588 |
|
589 /** |
|
590 * Load data from disk. |
|
591 * |
|
592 * @return Promise |
|
593 */ |
|
594 load: function () { |
|
595 return Task.spawn(function* () { |
|
596 // Loading replaces data. So reset data structures. |
|
597 this._data = { |
|
598 v: 1, |
|
599 crashes: new Map(), |
|
600 corruptDate: null, |
|
601 }; |
|
602 this._countsByDay = new Map(); |
|
603 |
|
604 try { |
|
605 let decoder = new TextDecoder(); |
|
606 let data = yield OS.File.read(this._storePath, {compression: "lz4"}); |
|
607 data = JSON.parse(decoder.decode(data)); |
|
608 |
|
609 if (data.corruptDate) { |
|
610 this._data.corruptDate = new Date(data.corruptDate); |
|
611 } |
|
612 |
|
613 // actualCounts is used to validate that the derived counts by |
|
614 // days stored in the payload matches up to actual data. |
|
615 let actualCounts = new Map(); |
|
616 |
|
617 for (let id in data.crashes) { |
|
618 let crash = data.crashes[id]; |
|
619 let denormalized = this._denormalize(crash); |
|
620 |
|
621 this._data.crashes.set(id, denormalized); |
|
622 |
|
623 let key = dateToDays(denormalized.crashDate) + "-" + denormalized.type; |
|
624 actualCounts.set(key, (actualCounts.get(key) || 0) + 1); |
|
625 } |
|
626 |
|
627 // The validation in this loop is arguably not necessary. We perform |
|
628 // it as a defense against unknown bugs. |
|
629 for (let dayKey in data.countsByDay) { |
|
630 let day = parseInt(dayKey, 10); |
|
631 for (let type in data.countsByDay[day]) { |
|
632 this._ensureCountsForDay(day); |
|
633 |
|
634 let count = data.countsByDay[day][type]; |
|
635 let key = day + "-" + type; |
|
636 |
|
637 // If the payload says we have data for a given day but we |
|
638 // don't, the payload is wrong. Ignore it. |
|
639 if (!actualCounts.has(key)) { |
|
640 continue; |
|
641 } |
|
642 |
|
643 // If we encountered more data in the payload than what the |
|
644 // data structure says, use the proper value. |
|
645 count = Math.max(count, actualCounts.get(key)); |
|
646 |
|
647 this._countsByDay.get(day).set(type, count); |
|
648 } |
|
649 } |
|
650 } catch (ex if ex instanceof OS.File.Error && ex.becauseNoSuchFile) { |
|
651 // Missing files (first use) are allowed. |
|
652 } catch (ex) { |
|
653 // If we can't load for any reason, mark a corrupt date in the instance |
|
654 // and swallow the error. |
|
655 // |
|
656 // The marking of a corrupted file is intentionally not persisted to |
|
657 // disk yet. Instead, we wait until the next save(). This is to give |
|
658 // non-permanent failures the opportunity to recover on their own. |
|
659 this._data.corruptDate = new Date(); |
|
660 } |
|
661 }.bind(this)); |
|
662 }, |
|
663 |
|
664 /** |
|
665 * Save data to disk. |
|
666 * |
|
667 * @return Promise<null> |
|
668 */ |
|
669 save: function () { |
|
670 return Task.spawn(function* () { |
|
671 if (!this._data) { |
|
672 return; |
|
673 } |
|
674 |
|
675 let normalized = { |
|
676 // The version should be incremented whenever the format |
|
677 // changes. |
|
678 v: 1, |
|
679 // Maps crash IDs to objects defining the crash. |
|
680 crashes: {}, |
|
681 // Maps days since UNIX epoch to objects mapping event types to |
|
682 // counts. This is a mirror of this._countsByDay. e.g. |
|
683 // { |
|
684 // 15000: { |
|
685 // "main-crash": 2, |
|
686 // "plugin-crash": 1 |
|
687 // } |
|
688 // } |
|
689 countsByDay: {}, |
|
690 |
|
691 // When the store was last corrupted. |
|
692 corruptDate: null, |
|
693 }; |
|
694 |
|
695 if (this._data.corruptDate) { |
|
696 normalized.corruptDate = this._data.corruptDate.getTime(); |
|
697 } |
|
698 |
|
699 for (let [id, crash] of this._data.crashes) { |
|
700 let c = this._normalize(crash); |
|
701 normalized.crashes[id] = c; |
|
702 } |
|
703 |
|
704 for (let [day, m] of this._countsByDay) { |
|
705 normalized.countsByDay[day] = {}; |
|
706 for (let [type, count] of m) { |
|
707 normalized.countsByDay[day][type] = count; |
|
708 } |
|
709 } |
|
710 |
|
711 let encoder = new TextEncoder(); |
|
712 let data = encoder.encode(JSON.stringify(normalized)); |
|
713 let size = yield OS.File.writeAtomic(this._storePath, data, { |
|
714 tmpPath: this._storePath + ".tmp", |
|
715 compression: "lz4"}); |
|
716 if (this._telemetrySizeKey) { |
|
717 Services.telemetry.getHistogramById(this._telemetrySizeKey).add(size); |
|
718 } |
|
719 }.bind(this)); |
|
720 }, |
|
721 |
|
722 /** |
|
723 * Normalize an object into one fit for serialization. |
|
724 * |
|
725 * This function along with _denormalize() serve to hack around the |
|
726 * default handling of Date JSON serialization because Date serialization |
|
727 * is undefined by JSON. |
|
728 * |
|
729 * Fields ending with "Date" are assumed to contain Date instances. |
|
730 * We convert these to milliseconds since epoch on output and back to |
|
731 * Date on input. |
|
732 */ |
|
733 _normalize: function (o) { |
|
734 let normalized = {}; |
|
735 |
|
736 for (let k in o) { |
|
737 let v = o[k]; |
|
738 if (v && k.endsWith("Date")) { |
|
739 normalized[k] = v.getTime(); |
|
740 } else { |
|
741 normalized[k] = v; |
|
742 } |
|
743 } |
|
744 |
|
745 return normalized; |
|
746 }, |
|
747 |
|
748 /** |
|
749 * Convert a serialized object back to its native form. |
|
750 */ |
|
751 _denormalize: function (o) { |
|
752 let n = {}; |
|
753 |
|
754 for (let k in o) { |
|
755 let v = o[k]; |
|
756 if (v && k.endsWith("Date")) { |
|
757 n[k] = new Date(parseInt(v, 10)); |
|
758 } else { |
|
759 n[k] = v; |
|
760 } |
|
761 } |
|
762 |
|
763 return n; |
|
764 }, |
|
765 |
|
766 /** |
|
767 * Prune old crash data. |
|
768 * |
|
769 * Crashes without recent activity are pruned from the store so the |
|
770 * size of the store is not unbounded. If there is activity on a crash, |
|
771 * that activity will keep the crash and all its data around for longer. |
|
772 * |
|
773 * @param date |
|
774 * (Date) The cutoff at which data will be pruned. If an entry |
|
775 * doesn't have data newer than this, it will be pruned. |
|
776 */ |
|
777 pruneOldCrashes: function (date) { |
|
778 for (let crash of this.crashes) { |
|
779 let newest = crash.newestDate; |
|
780 if (!newest || newest.getTime() < date.getTime()) { |
|
781 this._data.crashes.delete(crash.id); |
|
782 } |
|
783 } |
|
784 }, |
|
785 |
|
786 /** |
|
787 * Date the store was last corrupted and required a reset. |
|
788 * |
|
789 * May be null (no corruption has ever occurred) or a Date instance. |
|
790 */ |
|
791 get corruptDate() { |
|
792 return this._data.corruptDate; |
|
793 }, |
|
794 |
|
795 /** |
|
796 * The number of distinct crashes tracked. |
|
797 */ |
|
798 get crashesCount() { |
|
799 return this._data.crashes.size; |
|
800 }, |
|
801 |
|
802 /** |
|
803 * All crashes tracked. |
|
804 * |
|
805 * This is an array of CrashRecord. |
|
806 */ |
|
807 get crashes() { |
|
808 let crashes = []; |
|
809 for (let [id, crash] of this._data.crashes) { |
|
810 crashes.push(new CrashRecord(crash)); |
|
811 } |
|
812 |
|
813 return crashes; |
|
814 }, |
|
815 |
|
816 /** |
|
817 * Obtain a particular crash from its ID. |
|
818 * |
|
819 * A CrashRecord will be returned if the crash exists. null will be returned |
|
820 * if the crash is unknown. |
|
821 */ |
|
822 getCrash: function (id) { |
|
823 for (let crash of this.crashes) { |
|
824 if (crash.id == id) { |
|
825 return crash; |
|
826 } |
|
827 } |
|
828 |
|
829 return null; |
|
830 }, |
|
831 |
|
832 _ensureCountsForDay: function (day) { |
|
833 if (!this._countsByDay.has(day)) { |
|
834 this._countsByDay.set(day, new Map()); |
|
835 } |
|
836 }, |
|
837 |
|
838 /** |
|
839 * Ensure the crash record is present in storage. |
|
840 * |
|
841 * Returns the crash record if we're allowed to store it or null |
|
842 * if we've hit the high water mark. |
|
843 * |
|
844 * @param id |
|
845 * (string) The crash ID. |
|
846 * @param type |
|
847 * (string) One of the this.TYPE_* constants describing the crash type. |
|
848 * @param date |
|
849 * (Date) When this crash occurred. |
|
850 * |
|
851 * @return null | object crash record |
|
852 */ |
|
853 _ensureCrashRecord: function (id, type, date) { |
|
854 let day = dateToDays(date); |
|
855 this._ensureCountsForDay(day); |
|
856 |
|
857 let count = (this._countsByDay.get(day).get(type) || 0) + 1; |
|
858 this._countsByDay.get(day).set(type, count); |
|
859 |
|
860 if (count > this.HIGH_WATER_DAILY_THRESHOLD && type != this.TYPE_MAIN_CRASH) { |
|
861 return null; |
|
862 } |
|
863 |
|
864 if (!this._data.crashes.has(id)) { |
|
865 this._data.crashes.set(id, { |
|
866 id: id, |
|
867 type: type, |
|
868 crashDate: date, |
|
869 }); |
|
870 } |
|
871 |
|
872 let crash = this._data.crashes.get(id); |
|
873 crash.type = type; |
|
874 crash.date = date; |
|
875 |
|
876 return crash; |
|
877 }, |
|
878 |
|
879 /** |
|
880 * Record the occurrence of a crash in the main process. |
|
881 * |
|
882 * @param id (string) Crash ID. Likely a UUID. |
|
883 * @param date (Date) When the crash occurred. |
|
884 */ |
|
885 addMainProcessCrash: function (id, date) { |
|
886 this._ensureCrashRecord(id, this.TYPE_MAIN_CRASH, date); |
|
887 }, |
|
888 |
|
889 /** |
|
890 * Record the occurrence of a crash in a plugin process. |
|
891 * |
|
892 * @param id (string) Crash ID. Likely a UUID. |
|
893 * @param date (Date) When the crash occurred. |
|
894 */ |
|
895 addPluginCrash: function (id, date) { |
|
896 this._ensureCrashRecord(id, this.TYPE_PLUGIN_CRASH, date); |
|
897 }, |
|
898 |
|
899 /** |
|
900 * Record the occurrence of a hang in a plugin process. |
|
901 * |
|
902 * @param id (string) Crash ID. Likely a UUID. |
|
903 * @param date (Date) When the hang was reported. |
|
904 */ |
|
905 addPluginHang: function (id, date) { |
|
906 this._ensureCrashRecord(id, this.TYPE_PLUGIN_HANG, date); |
|
907 }, |
|
908 |
|
909 get mainProcessCrashes() { |
|
910 let crashes = []; |
|
911 for (let crash of this.crashes) { |
|
912 if (crash.isMainProcessCrash) { |
|
913 crashes.push(crash); |
|
914 } |
|
915 } |
|
916 |
|
917 return crashes; |
|
918 }, |
|
919 |
|
920 get pluginCrashes() { |
|
921 let crashes = []; |
|
922 for (let crash of this.crashes) { |
|
923 if (crash.isPluginCrash) { |
|
924 crashes.push(crash); |
|
925 } |
|
926 } |
|
927 |
|
928 return crashes; |
|
929 }, |
|
930 |
|
931 get pluginHangs() { |
|
932 let crashes = []; |
|
933 for (let crash of this.crashes) { |
|
934 if (crash.isPluginHang) { |
|
935 crashes.push(crash); |
|
936 } |
|
937 } |
|
938 |
|
939 return crashes; |
|
940 }, |
|
941 }); |
|
942 |
|
943 /** |
|
944 * Represents an individual crash with metadata. |
|
945 * |
|
946 * This is a wrapper around the low-level anonymous JS objects that define |
|
947 * crashes. It exposes a consistent and helpful API. |
|
948 * |
|
949 * Instances of this type should only be constructured inside this module, |
|
950 * not externally. The constructor is not considered a public API. |
|
951 * |
|
952 * @param o (object) |
|
953 * The crash's entry from the CrashStore. |
|
954 */ |
|
955 function CrashRecord(o) { |
|
956 this._o = o; |
|
957 } |
|
958 |
|
959 CrashRecord.prototype = Object.freeze({ |
|
960 get id() { |
|
961 return this._o.id; |
|
962 }, |
|
963 |
|
964 get crashDate() { |
|
965 return this._o.crashDate; |
|
966 }, |
|
967 |
|
968 /** |
|
969 * Obtain the newest date in this record. |
|
970 * |
|
971 * This is a convenience getter. The returned value is used to determine when |
|
972 * to expire a record. |
|
973 */ |
|
974 get newestDate() { |
|
975 // We currently only have 1 date, so this is easy. |
|
976 return this._o.crashDate; |
|
977 }, |
|
978 |
|
979 get oldestDate() { |
|
980 return this._o.crashDate; |
|
981 }, |
|
982 |
|
983 get type() { |
|
984 return this._o.type; |
|
985 }, |
|
986 |
|
987 get isMainProcessCrash() { |
|
988 return this._o.type == CrashStore.prototype.TYPE_MAIN_CRASH; |
|
989 }, |
|
990 |
|
991 get isPluginCrash() { |
|
992 return this._o.type == CrashStore.prototype.TYPE_PLUGIN_CRASH; |
|
993 }, |
|
994 |
|
995 get isPluginHang() { |
|
996 return this._o.type == CrashStore.prototype.TYPE_PLUGIN_HANG; |
|
997 }, |
|
998 }); |
|
999 |
|
1000 /** |
|
1001 * Obtain the global CrashManager instance used by the running application. |
|
1002 * |
|
1003 * CrashManager is likely only ever instantiated once per application lifetime. |
|
1004 * The main reason it's implemented as a reusable type is to facilitate testing. |
|
1005 */ |
|
1006 XPCOMUtils.defineLazyGetter(this.CrashManager, "Singleton", function () { |
|
1007 if (gCrashManager) { |
|
1008 return gCrashManager; |
|
1009 } |
|
1010 |
|
1011 let crPath = OS.Path.join(OS.Constants.Path.userApplicationDataDir, |
|
1012 "Crash Reports"); |
|
1013 let storePath = OS.Path.join(OS.Constants.Path.profileDir, "crashes"); |
|
1014 |
|
1015 gCrashManager = new CrashManager({ |
|
1016 pendingDumpsDir: OS.Path.join(crPath, "pending"), |
|
1017 submittedDumpsDir: OS.Path.join(crPath, "submitted"), |
|
1018 eventsDirs: [OS.Path.join(crPath, "events"), OS.Path.join(storePath, "events")], |
|
1019 storeDir: storePath, |
|
1020 telemetryStoreSizeKey: "CRASH_STORE_COMPRESSED_BYTES", |
|
1021 }); |
|
1022 |
|
1023 // Automatically aggregate event files shortly after startup. This |
|
1024 // ensures it happens with some frequency. |
|
1025 // |
|
1026 // There are performance considerations here. While this is doing |
|
1027 // work and could negatively impact performance, the amount of work |
|
1028 // is kept small per run by periodically aggregating event files. |
|
1029 // Furthermore, well-behaving installs should not have much work |
|
1030 // here to do. If there is a lot of work, that install has bigger |
|
1031 // issues beyond reduced performance near startup. |
|
1032 gCrashManager.scheduleMaintenance(AGGREGATE_STARTUP_DELAY_MS); |
|
1033 |
|
1034 return gCrashManager; |
|
1035 }); |