1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/storage/test/test_deadlock_detector.cpp Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,605 @@
1.4 +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*-
1.5 + * vim: sw=4 ts=4 et :
1.6 + * This Source Code Form is subject to the terms of the Mozilla Public
1.7 + * License, v. 2.0. If a copy of the MPL was not distributed with this
1.8 + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
1.9 +
1.10 +/**
1.11 + * Note: This file is a copy of xpcom/tests/TestDeadlockDetector.cpp, but all
1.12 + * mutexes were turned into SQLiteMutexes.
1.13 + */
1.14 +
1.15 +#include "prenv.h"
1.16 +#include "prerror.h"
1.17 +#include "prio.h"
1.18 +#include "prproces.h"
1.19 +
1.20 +#include "nsMemory.h"
1.21 +
1.22 +#include "mozilla/CondVar.h"
1.23 +#include "mozilla/ReentrantMonitor.h"
1.24 +#include "SQLiteMutex.h"
1.25 +
1.26 +#include "TestHarness.h"
1.27 +
1.28 +using namespace mozilla;
1.29 +
1.30 +/**
1.31 + * Helper class to allocate a sqlite3_mutex for our SQLiteMutex. Also makes
1.32 + * keeping the test files in sync easier.
1.33 + */
1.34 +class TestMutex : public mozilla::storage::SQLiteMutex
1.35 +{
1.36 +public:
1.37 + TestMutex(const char* aName)
1.38 + : mozilla::storage::SQLiteMutex(aName)
1.39 + , mInner(sqlite3_mutex_alloc(SQLITE_MUTEX_FAST))
1.40 + {
1.41 + NS_ASSERTION(mInner, "could not allocate a sqlite3_mutex");
1.42 + initWithMutex(mInner);
1.43 + }
1.44 +
1.45 + ~TestMutex()
1.46 + {
1.47 + sqlite3_mutex_free(mInner);
1.48 + }
1.49 +
1.50 + void Lock()
1.51 + {
1.52 + lock();
1.53 + }
1.54 +
1.55 + void Unlock()
1.56 + {
1.57 + unlock();
1.58 + }
1.59 +private:
1.60 + sqlite3_mutex *mInner;
1.61 +};
1.62 +
1.63 +static PRThread*
1.64 +spawn(void (*run)(void*), void* arg)
1.65 +{
1.66 + return PR_CreateThread(PR_SYSTEM_THREAD,
1.67 + run,
1.68 + arg,
1.69 + PR_PRIORITY_NORMAL,
1.70 + PR_GLOBAL_THREAD,
1.71 + PR_JOINABLE_THREAD,
1.72 + 0);
1.73 +}
1.74 +
1.75 +#define PASS() \
1.76 + do { \
1.77 + passed(__FUNCTION__); \
1.78 + return NS_OK; \
1.79 + } while (0)
1.80 +
1.81 +#define FAIL(why) \
1.82 + do { \
1.83 + fail("%s | %s - %s", __FILE__, __FUNCTION__, why); \
1.84 + return NS_ERROR_FAILURE; \
1.85 + } while (0)
1.86 +
1.87 +//-----------------------------------------------------------------------------
1.88 +
1.89 +static const char* sPathToThisBinary;
1.90 +static const char* sAssertBehaviorEnv = "XPCOM_DEBUG_BREAK=abort";
1.91 +
1.92 +class Subprocess
1.93 +{
1.94 +public:
1.95 + // not available until process finishes
1.96 + int32_t mExitCode;
1.97 + nsCString mStdout;
1.98 + nsCString mStderr;
1.99 +
1.100 + Subprocess(const char* aTestName) {
1.101 + // set up stdio redirection
1.102 + PRFileDesc* readStdin; PRFileDesc* writeStdin;
1.103 + PRFileDesc* readStdout; PRFileDesc* writeStdout;
1.104 + PRFileDesc* readStderr; PRFileDesc* writeStderr;
1.105 + PRProcessAttr* pattr = PR_NewProcessAttr();
1.106 +
1.107 + NS_ASSERTION(pattr, "couldn't allocate process attrs");
1.108 +
1.109 + NS_ASSERTION(PR_SUCCESS == PR_CreatePipe(&readStdin, &writeStdin),
1.110 + "couldn't create child stdin pipe");
1.111 + NS_ASSERTION(PR_SUCCESS == PR_SetFDInheritable(readStdin, true),
1.112 + "couldn't set child stdin inheritable");
1.113 + PR_ProcessAttrSetStdioRedirect(pattr, PR_StandardInput, readStdin);
1.114 +
1.115 + NS_ASSERTION(PR_SUCCESS == PR_CreatePipe(&readStdout, &writeStdout),
1.116 + "couldn't create child stdout pipe");
1.117 + NS_ASSERTION(PR_SUCCESS == PR_SetFDInheritable(writeStdout, true),
1.118 + "couldn't set child stdout inheritable");
1.119 + PR_ProcessAttrSetStdioRedirect(pattr, PR_StandardOutput, writeStdout);
1.120 +
1.121 + NS_ASSERTION(PR_SUCCESS == PR_CreatePipe(&readStderr, &writeStderr),
1.122 + "couldn't create child stderr pipe");
1.123 + NS_ASSERTION(PR_SUCCESS == PR_SetFDInheritable(writeStderr, true),
1.124 + "couldn't set child stderr inheritable");
1.125 + PR_ProcessAttrSetStdioRedirect(pattr, PR_StandardError, writeStderr);
1.126 +
1.127 + // set up argv with test name to run
1.128 + char* const newArgv[3] = {
1.129 + strdup(sPathToThisBinary),
1.130 + strdup(aTestName),
1.131 + 0
1.132 + };
1.133 +
1.134 + // make sure the child will abort if an assertion fails
1.135 + NS_ASSERTION(PR_SUCCESS == PR_SetEnv(sAssertBehaviorEnv),
1.136 + "couldn't set XPCOM_DEBUG_BREAK env var");
1.137 +
1.138 + PRProcess* proc;
1.139 + NS_ASSERTION(proc = PR_CreateProcess(sPathToThisBinary,
1.140 + newArgv,
1.141 + 0, // inherit environment
1.142 + pattr),
1.143 + "couldn't create process");
1.144 + PR_Close(readStdin);
1.145 + PR_Close(writeStdout);
1.146 + PR_Close(writeStderr);
1.147 +
1.148 + mProc = proc;
1.149 + mStdinfd = writeStdin;
1.150 + mStdoutfd = readStdout;
1.151 + mStderrfd = readStderr;
1.152 +
1.153 + free(newArgv[0]);
1.154 + free(newArgv[1]);
1.155 + PR_DestroyProcessAttr(pattr);
1.156 + }
1.157 +
1.158 + void RunToCompletion(uint32_t aWaitMs)
1.159 + {
1.160 + PR_Close(mStdinfd);
1.161 +
1.162 + PRPollDesc pollfds[2];
1.163 + int32_t nfds;
1.164 + bool stdoutOpen = true, stderrOpen = true;
1.165 + char buf[4096];
1.166 + int32_t len;
1.167 +
1.168 + PRIntervalTime now = PR_IntervalNow();
1.169 + PRIntervalTime deadline = now + PR_MillisecondsToInterval(aWaitMs);
1.170 +
1.171 + while ((stdoutOpen || stderrOpen) && now < deadline) {
1.172 + nfds = 0;
1.173 + if (stdoutOpen) {
1.174 + pollfds[nfds].fd = mStdoutfd;
1.175 + pollfds[nfds].in_flags = PR_POLL_READ;
1.176 + pollfds[nfds].out_flags = 0;
1.177 + ++nfds;
1.178 + }
1.179 + if (stderrOpen) {
1.180 + pollfds[nfds].fd = mStderrfd;
1.181 + pollfds[nfds].in_flags = PR_POLL_READ;
1.182 + pollfds[nfds].out_flags = 0;
1.183 + ++nfds;
1.184 + }
1.185 +
1.186 + int32_t rv = PR_Poll(pollfds, nfds, deadline - now);
1.187 + NS_ASSERTION(0 <= rv, PR_ErrorToName(PR_GetError()));
1.188 +
1.189 + if (0 == rv) { // timeout
1.190 + fputs("(timed out!)\n", stderr);
1.191 + Finish(false); // abnormal
1.192 + return;
1.193 + }
1.194 +
1.195 + for (int32_t i = 0; i < nfds; ++i) {
1.196 + if (!pollfds[i].out_flags)
1.197 + continue;
1.198 +
1.199 + bool isStdout = mStdoutfd == pollfds[i].fd;
1.200 +
1.201 + if (PR_POLL_READ & pollfds[i].out_flags) {
1.202 + len = PR_Read(pollfds[i].fd, buf, sizeof(buf) - 1);
1.203 + NS_ASSERTION(0 <= len, PR_ErrorToName(PR_GetError()));
1.204 + }
1.205 + else if (PR_POLL_HUP & pollfds[i].out_flags) {
1.206 + len = 0;
1.207 + }
1.208 + else {
1.209 + NS_ERROR(PR_ErrorToName(PR_GetError()));
1.210 + }
1.211 +
1.212 + if (0 < len) {
1.213 + buf[len] = '\0';
1.214 + if (isStdout)
1.215 + mStdout += buf;
1.216 + else
1.217 + mStderr += buf;
1.218 + }
1.219 + else if (isStdout) {
1.220 + stdoutOpen = false;
1.221 + }
1.222 + else {
1.223 + stderrOpen = false;
1.224 + }
1.225 + }
1.226 +
1.227 + now = PR_IntervalNow();
1.228 + }
1.229 +
1.230 + if (stdoutOpen)
1.231 + fputs("(stdout still open!)\n", stderr);
1.232 + if (stderrOpen)
1.233 + fputs("(stderr still open!)\n", stderr);
1.234 + if (now > deadline)
1.235 + fputs("(timed out!)\n", stderr);
1.236 +
1.237 + Finish(!stdoutOpen && !stderrOpen && now <= deadline);
1.238 + }
1.239 +
1.240 +private:
1.241 + void Finish(bool normalExit) {
1.242 + if (!normalExit) {
1.243 + PR_KillProcess(mProc);
1.244 + mExitCode = -1;
1.245 + int32_t dummy;
1.246 + PR_WaitProcess(mProc, &dummy);
1.247 + }
1.248 + else {
1.249 + PR_WaitProcess(mProc, &mExitCode); // this had better not block ...
1.250 + }
1.251 +
1.252 + PR_Close(mStdoutfd);
1.253 + PR_Close(mStderrfd);
1.254 + }
1.255 +
1.256 + PRProcess* mProc;
1.257 + PRFileDesc* mStdinfd; // writeable
1.258 + PRFileDesc* mStdoutfd; // readable
1.259 + PRFileDesc* mStderrfd; // readable
1.260 +};
1.261 +
1.262 +//-----------------------------------------------------------------------------
1.263 +// Harness for checking detector errors
1.264 +bool
1.265 +CheckForDeadlock(const char* test, const char* const* findTokens)
1.266 +{
1.267 + Subprocess proc(test);
1.268 + proc.RunToCompletion(5000);
1.269 +
1.270 + if (0 == proc.mExitCode)
1.271 + return false;
1.272 +
1.273 + int32_t idx = 0;
1.274 + for (const char* const* tp = findTokens; *tp; ++tp) {
1.275 + const char* const token = *tp;
1.276 +#ifdef MOZILLA_INTERNAL_API
1.277 + idx = proc.mStderr.Find(token, false, idx);
1.278 +#else
1.279 + nsCString tokenCString(token);
1.280 + idx = proc.mStderr.Find(tokenCString, idx);
1.281 +#endif
1.282 + if (-1 == idx) {
1.283 + printf("(missed token '%s' in output)\n", token);
1.284 + puts("----------------------------------\n");
1.285 + puts(proc.mStderr.get());
1.286 + puts("----------------------------------\n");
1.287 + return false;
1.288 + }
1.289 + idx += strlen(token);
1.290 + }
1.291 +
1.292 + return true;
1.293 +}
1.294 +
1.295 +//-----------------------------------------------------------------------------
1.296 +// Single-threaded sanity tests
1.297 +
1.298 +// Stupidest possible deadlock.
1.299 +int
1.300 +Sanity_Child()
1.301 +{
1.302 + TestMutex m1("dd.sanity.m1");
1.303 + m1.Lock();
1.304 + m1.Lock();
1.305 + return 0; // not reached
1.306 +}
1.307 +
1.308 +nsresult
1.309 +Sanity()
1.310 +{
1.311 + const char* const tokens[] = {
1.312 + "###!!! ERROR: Potential deadlock detected",
1.313 + "=== Cyclical dependency starts at\n--- Mutex : dd.sanity.m1",
1.314 + "=== Cycle completed at\n--- Mutex : dd.sanity.m1",
1.315 + "###!!! Deadlock may happen NOW!", // better catch these easy cases...
1.316 + "###!!! ASSERTION: Potential deadlock detected",
1.317 + 0
1.318 + };
1.319 + if (CheckForDeadlock("Sanity", tokens)) {
1.320 + PASS();
1.321 + } else {
1.322 + FAIL("deadlock not detected");
1.323 + }
1.324 +}
1.325 +
1.326 +// Slightly less stupid deadlock.
1.327 +int
1.328 +Sanity2_Child()
1.329 +{
1.330 + TestMutex m1("dd.sanity2.m1");
1.331 + TestMutex m2("dd.sanity2.m2");
1.332 + m1.Lock();
1.333 + m2.Lock();
1.334 + m1.Lock();
1.335 + return 0; // not reached
1.336 +}
1.337 +
1.338 +nsresult
1.339 +Sanity2()
1.340 +{
1.341 + const char* const tokens[] = {
1.342 + "###!!! ERROR: Potential deadlock detected",
1.343 + "=== Cyclical dependency starts at\n--- Mutex : dd.sanity2.m1",
1.344 + "--- Next dependency:\n--- Mutex : dd.sanity2.m2",
1.345 + "=== Cycle completed at\n--- Mutex : dd.sanity2.m1",
1.346 + "###!!! Deadlock may happen NOW!", // better catch these easy cases...
1.347 + "###!!! ASSERTION: Potential deadlock detected",
1.348 + 0
1.349 + };
1.350 + if (CheckForDeadlock("Sanity2", tokens)) {
1.351 + PASS();
1.352 + } else {
1.353 + FAIL("deadlock not detected");
1.354 + }
1.355 +}
1.356 +
1.357 +
1.358 +int
1.359 +Sanity3_Child()
1.360 +{
1.361 + TestMutex m1("dd.sanity3.m1");
1.362 + TestMutex m2("dd.sanity3.m2");
1.363 + TestMutex m3("dd.sanity3.m3");
1.364 + TestMutex m4("dd.sanity3.m4");
1.365 +
1.366 + m1.Lock();
1.367 + m2.Lock();
1.368 + m3.Lock();
1.369 + m4.Lock();
1.370 + m4.Unlock();
1.371 + m3.Unlock();
1.372 + m2.Unlock();
1.373 + m1.Unlock();
1.374 +
1.375 + m4.Lock();
1.376 + m1.Lock();
1.377 + return 0;
1.378 +}
1.379 +
1.380 +nsresult
1.381 +Sanity3()
1.382 +{
1.383 + const char* const tokens[] = {
1.384 + "###!!! ERROR: Potential deadlock detected",
1.385 + "=== Cyclical dependency starts at\n--- Mutex : dd.sanity3.m1",
1.386 + "--- Next dependency:\n--- Mutex : dd.sanity3.m2",
1.387 + "--- Next dependency:\n--- Mutex : dd.sanity3.m3",
1.388 + "--- Next dependency:\n--- Mutex : dd.sanity3.m4",
1.389 + "=== Cycle completed at\n--- Mutex : dd.sanity3.m1",
1.390 + "###!!! ASSERTION: Potential deadlock detected",
1.391 + 0
1.392 + };
1.393 + if (CheckForDeadlock("Sanity3", tokens)) {
1.394 + PASS();
1.395 + } else {
1.396 + FAIL("deadlock not detected");
1.397 + }
1.398 +}
1.399 +
1.400 +
1.401 +int
1.402 +Sanity4_Child()
1.403 +{
1.404 + mozilla::ReentrantMonitor m1("dd.sanity4.m1");
1.405 + TestMutex m2("dd.sanity4.m2");
1.406 + m1.Enter();
1.407 + m2.Lock();
1.408 + m1.Enter();
1.409 + return 0;
1.410 +}
1.411 +
1.412 +nsresult
1.413 +Sanity4()
1.414 +{
1.415 + const char* const tokens[] = {
1.416 + "Re-entering ReentrantMonitor after acquiring other resources",
1.417 + "###!!! ERROR: Potential deadlock detected",
1.418 + "=== Cyclical dependency starts at\n--- ReentrantMonitor : dd.sanity4.m1",
1.419 + "--- Next dependency:\n--- Mutex : dd.sanity4.m2",
1.420 + "=== Cycle completed at\n--- ReentrantMonitor : dd.sanity4.m1",
1.421 + "###!!! ASSERTION: Potential deadlock detected",
1.422 + 0
1.423 + };
1.424 + if (CheckForDeadlock("Sanity4", tokens)) {
1.425 + PASS();
1.426 + } else {
1.427 + FAIL("deadlock not detected");
1.428 + }
1.429 +}
1.430 +
1.431 +//-----------------------------------------------------------------------------
1.432 +// Multithreaded tests
1.433 +
1.434 +TestMutex* ttM1;
1.435 +TestMutex* ttM2;
1.436 +
1.437 +static void
1.438 +TwoThreads_thread(void* arg)
1.439 +{
1.440 + int32_t m1First = NS_PTR_TO_INT32(arg);
1.441 + if (m1First) {
1.442 + ttM1->Lock();
1.443 + ttM2->Lock();
1.444 + ttM2->Unlock();
1.445 + ttM1->Unlock();
1.446 + }
1.447 + else {
1.448 + ttM2->Lock();
1.449 + ttM1->Lock();
1.450 + ttM1->Unlock();
1.451 + ttM2->Unlock();
1.452 + }
1.453 +}
1.454 +
1.455 +int
1.456 +TwoThreads_Child()
1.457 +{
1.458 + ttM1 = new TestMutex("dd.twothreads.m1");
1.459 + ttM2 = new TestMutex("dd.twothreads.m2");
1.460 + if (!ttM1 || !ttM2)
1.461 + NS_RUNTIMEABORT("couldn't allocate mutexes");
1.462 +
1.463 + PRThread* t1 = spawn(TwoThreads_thread, (void*) 0);
1.464 + PR_JoinThread(t1);
1.465 +
1.466 + PRThread* t2 = spawn(TwoThreads_thread, (void*) 1);
1.467 + PR_JoinThread(t2);
1.468 +
1.469 + return 0;
1.470 +}
1.471 +
1.472 +nsresult
1.473 +TwoThreads()
1.474 +{
1.475 + const char* const tokens[] = {
1.476 + "###!!! ERROR: Potential deadlock detected",
1.477 + "=== Cyclical dependency starts at\n--- Mutex : dd.twothreads.m2",
1.478 + "--- Next dependency:\n--- Mutex : dd.twothreads.m1",
1.479 + "=== Cycle completed at\n--- Mutex : dd.twothreads.m2",
1.480 + "###!!! ASSERTION: Potential deadlock detected",
1.481 + 0
1.482 + };
1.483 +
1.484 + if (CheckForDeadlock("TwoThreads", tokens)) {
1.485 + PASS();
1.486 + } else {
1.487 + FAIL("deadlock not detected");
1.488 + }
1.489 +}
1.490 +
1.491 +
1.492 +TestMutex* cndMs[4];
1.493 +const uint32_t K = 100000;
1.494 +
1.495 +static void
1.496 +ContentionNoDeadlock_thread(void* arg)
1.497 +{
1.498 + int32_t starti = NS_PTR_TO_INT32(arg);
1.499 +
1.500 + for (uint32_t k = 0; k < K; ++k) {
1.501 + for (int32_t i = starti; i < (int32_t) ArrayLength(cndMs); ++i)
1.502 + cndMs[i]->Lock();
1.503 + // comment out the next two lines for deadlocking fun!
1.504 + for (int32_t i = ArrayLength(cndMs) - 1; i >= starti; --i)
1.505 + cndMs[i]->Unlock();
1.506 +
1.507 + starti = (starti + 1) % 3;
1.508 + }
1.509 +}
1.510 +
1.511 +int
1.512 +ContentionNoDeadlock_Child()
1.513 +{
1.514 + PRThread* threads[3];
1.515 +
1.516 + for (uint32_t i = 0; i < ArrayLength(cndMs); ++i)
1.517 + cndMs[i] = new TestMutex("dd.cnd.ms");
1.518 +
1.519 + for (int32_t i = 0; i < (int32_t) ArrayLength(threads); ++i)
1.520 + threads[i] = spawn(ContentionNoDeadlock_thread, NS_INT32_TO_PTR(i));
1.521 +
1.522 + for (uint32_t i = 0; i < ArrayLength(threads); ++i)
1.523 + PR_JoinThread(threads[i]);
1.524 +
1.525 + for (uint32_t i = 0; i < ArrayLength(cndMs); ++i)
1.526 + delete cndMs[i];
1.527 +
1.528 + return 0;
1.529 +}
1.530 +
1.531 +nsresult
1.532 +ContentionNoDeadlock()
1.533 +{
1.534 + const char * func = __func__;
1.535 + Subprocess proc(func);
1.536 + proc.RunToCompletion(60000);
1.537 + if (0 != proc.mExitCode) {
1.538 + printf("(expected 0 == return code, got %d)\n", proc.mExitCode);
1.539 + puts("(output)\n----------------------------------\n");
1.540 + puts(proc.mStdout.get());
1.541 + puts("----------------------------------\n");
1.542 + puts("(error output)\n----------------------------------\n");
1.543 + puts(proc.mStderr.get());
1.544 + puts("----------------------------------\n");
1.545 +
1.546 + FAIL("deadlock");
1.547 + }
1.548 + PASS();
1.549 +}
1.550 +
1.551 +
1.552 +
1.553 +//-----------------------------------------------------------------------------
1.554 +
1.555 +int
1.556 +main(int argc, char** argv)
1.557 +{
1.558 + if (1 < argc) {
1.559 + // XXX can we run w/o scoped XPCOM?
1.560 + const char* test = argv[1];
1.561 + ScopedXPCOM xpcom(test);
1.562 + if (xpcom.failed())
1.563 + return 1;
1.564 +
1.565 + // running in a spawned process. call the specificed child function.
1.566 + if (!strcmp("Sanity", test))
1.567 + return Sanity_Child();
1.568 + if (!strcmp("Sanity2", test))
1.569 + return Sanity2_Child();
1.570 + if (!strcmp("Sanity3", test))
1.571 + return Sanity3_Child();
1.572 + if (!strcmp("Sanity4", test))
1.573 + return Sanity4_Child();
1.574 +
1.575 + if (!strcmp("TwoThreads", test))
1.576 + return TwoThreads_Child();
1.577 + if (!strcmp("ContentionNoDeadlock", test))
1.578 + return ContentionNoDeadlock_Child();
1.579 +
1.580 + fail("%s | %s - unknown child test", __FILE__, __FUNCTION__);
1.581 + return 1;
1.582 + }
1.583 +
1.584 + ScopedXPCOM xpcom("Storage deadlock detector correctness (" __FILE__ ")");
1.585 + if (xpcom.failed())
1.586 + return 1;
1.587 +
1.588 + // in the first invocation of this process. we will be the "driver".
1.589 + int rv = 0;
1.590 +
1.591 + sPathToThisBinary = argv[0];
1.592 +
1.593 + if (NS_FAILED(Sanity()))
1.594 + rv = 1;
1.595 + if (NS_FAILED(Sanity2()))
1.596 + rv = 1;
1.597 + if (NS_FAILED(Sanity3()))
1.598 + rv = 1;
1.599 + if (NS_FAILED(Sanity4()))
1.600 + rv = 1;
1.601 +
1.602 + if (NS_FAILED(TwoThreads()))
1.603 + rv = 1;
1.604 + if (NS_FAILED(ContentionNoDeadlock()))
1.605 + rv = 1;
1.606 +
1.607 + return rv;
1.608 +}
