michael@0: /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
michael@0:  * You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: #include <map>
michael@0: #include <memory>
michael@0: 
michael@0: #include <dlfcn.h>
michael@0: #include <errno.h>
michael@0: #include <fcntl.h>
michael@0: #include <setjmp.h>
michael@0: #include <signal.h>
michael@0: #include <poll.h>
michael@0: #include <pthread.h>
michael@0: #include <alloca.h>
michael@0: #include <sys/epoll.h>
michael@0: #include <sys/mman.h>
michael@0: #include <sys/prctl.h>
michael@0: #include <sys/types.h>
michael@0: #include <sys/socket.h>
michael@0: #include <sys/stat.h>
michael@0: #include <sys/syscall.h>
michael@0: #include <vector>
michael@0: 
michael@0: #include "mozilla/LinkedList.h"
michael@0: #include "Nuwa.h"
michael@0: 
michael@0: using namespace mozilla;
michael@0: 
michael@0: extern "C" MFBT_API int tgkill(pid_t tgid, pid_t tid, int signalno) {
michael@0:   return syscall(__NR_tgkill, tgid, tid, signalno);
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Provides the wrappers to a selected set of pthread and system-level functions
michael@0:  * as the basis for implementing Zygote-like preforking mechanism.
michael@0:  */
michael@0: 
michael@0: /**
michael@0:  * Real functions for the wrappers.
michael@0:  */
michael@0: extern "C" {
michael@0: int __real_pthread_create(pthread_t *thread,
michael@0:                           const pthread_attr_t *attr,
michael@0:                           void *(*start_routine) (void *),
michael@0:                           void *arg);
michael@0: int __real_pthread_key_create(pthread_key_t *key, void (*destructor)(void*));
michael@0: int __real_pthread_key_delete(pthread_key_t key);
michael@0: pthread_t __real_pthread_self();
michael@0: int __real_pthread_join(pthread_t thread, void **retval);
michael@0: int __real_epoll_wait(int epfd,
michael@0:                       struct epoll_event *events,
michael@0:                       int maxevents,
michael@0:                       int timeout);
michael@0: int __real_pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mtx);
michael@0: int __real_pthread_cond_timedwait(pthread_cond_t *cond,
michael@0:                                   pthread_mutex_t *mtx,
michael@0:                                   const struct timespec *abstime);
michael@0: int __real___pthread_cond_timedwait(pthread_cond_t *cond,
michael@0:                                     pthread_mutex_t *mtx,
michael@0:                                     const struct timespec *abstime,
michael@0:                                     clockid_t clock);
michael@0: int __real_pthread_mutex_lock(pthread_mutex_t *mtx);
michael@0: int __real_poll(struct pollfd *fds, nfds_t nfds, int timeout);
michael@0: int __real_epoll_create(int size);
michael@0: int __real_socketpair(int domain, int type, int protocol, int sv[2]);
michael@0: int __real_pipe2(int __pipedes[2], int flags);
michael@0: int __real_pipe(int __pipedes[2]);
michael@0: int __real_epoll_ctl(int aEpollFd, int aOp, int aFd, struct epoll_event *aEvent);
michael@0: int __real_close(int aFd);
michael@0: }
michael@0: 
michael@0: #define REAL(s) __real_##s
michael@0: 
michael@0: /**
michael@0:  * A Nuwa process is started by preparing.  After preparing, it waits
michael@0:  * for all threads becoming frozen. Then, it is ready while all
michael@0:  * threads are frozen.
michael@0:  */
michael@0: static bool sIsNuwaProcess = false; // This process is a Nuwa process.
michael@0: static bool sIsFreezing = false; // Waiting for all threads getting frozen.
michael@0: static bool sNuwaReady = false;  // Nuwa process is ready.
michael@0: static bool sNuwaPendingSpawn = false; // Are there any pending spawn requests?
michael@0: static bool sNuwaForking = false;
michael@0: 
michael@0: // Fds of transports of top level protocols.
michael@0: static NuwaProtoFdInfo sProtoFdInfos[NUWA_TOPLEVEL_MAX];
michael@0: static int sProtoFdInfosSize = 0;
michael@0: 
michael@0: template <typename T>
michael@0: struct LibcAllocator: public std::allocator<T>
michael@0: {
michael@0:   LibcAllocator()
michael@0:   {
michael@0:     void* libcHandle = dlopen("libc.so", RTLD_LAZY);
michael@0:     mMallocImpl = reinterpret_cast<void*(*)(size_t)>(dlsym(libcHandle, "malloc"));
michael@0:     mFreeImpl = reinterpret_cast<void(*)(void*)>(dlsym(libcHandle, "free"));
michael@0: 
michael@0:     if (!(mMallocImpl && mFreeImpl)) {
michael@0:       // libc should be available, or we'll deadlock in using TLSInfoList.
michael@0:       abort();
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   inline typename std::allocator<T>::pointer
michael@0:   allocate(typename std::allocator<T>::size_type n,
michael@0:            const void * = 0)
michael@0:   {
michael@0:     return reinterpret_cast<T *>(mMallocImpl(sizeof(T) * n));
michael@0:   }
michael@0: 
michael@0:   inline void
michael@0:   deallocate(typename std::allocator<T>::pointer p,
michael@0:              typename std::allocator<T>::size_type n)
michael@0:   {
michael@0:     mFreeImpl(p);
michael@0:   }
michael@0: 
michael@0:   template<typename U>
michael@0:   struct rebind
michael@0:   {
michael@0:     typedef LibcAllocator<U> other;
michael@0:   };
michael@0: private:
michael@0:   void* (*mMallocImpl)(size_t);
michael@0:   void (*mFreeImpl)(void*);
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * TLSInfoList should use malloc() and free() in libc to avoid the deadlock that
michael@0:  * jemalloc calls into __wrap_pthread_mutex_lock() and then deadlocks while
michael@0:  * the same thread already acquired sThreadCountLock.
michael@0:  */
michael@0: typedef std::vector<std::pair<pthread_key_t, void *>,
michael@0:                     LibcAllocator<std::pair<pthread_key_t, void *> > >
michael@0: TLSInfoList;
michael@0: 
michael@0: /**
michael@0:  * Return the system's page size
michael@0:  */
michael@0: static size_t getPageSize(void) {
michael@0: #ifdef HAVE_GETPAGESIZE
michael@0:   return getpagesize();
michael@0: #elif defined(_SC_PAGESIZE)
michael@0:   return sysconf(_SC_PAGESIZE);
michael@0: #elif defined(PAGE_SIZE)
michael@0:   return PAGE_SIZE;
michael@0: #else
michael@0:   #warning "Hard-coding page size to 4096 bytes"
michael@0:   return 4096
michael@0: #endif
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Align the pointer to the next page boundary unless it's already aligned
michael@0:  */
michael@0: static uintptr_t ceilToPage(uintptr_t aPtr) {
michael@0:   size_t pageSize = getPageSize();
michael@0: 
michael@0:   return ((aPtr + pageSize - 1) / pageSize) * pageSize;
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * The stack size is chosen carefully so the frozen threads doesn't consume too
michael@0:  * much memory in the Nuwa process. The threads shouldn't run deep recursive
michael@0:  * methods or do large allocations on the stack to avoid stack overflow.
michael@0:  */
michael@0: #ifndef NUWA_STACK_SIZE
michael@0: #define NUWA_STACK_SIZE (1024 * 128)
michael@0: #endif
michael@0: 
michael@0: #define NATIVE_THREAD_NAME_LENGTH 16
michael@0: 
michael@0: struct thread_info : public mozilla::LinkedListElement<thread_info> {
michael@0:   pthread_t origThreadID;
michael@0:   pthread_t recreatedThreadID;
michael@0:   pthread_attr_t threadAttr;
michael@0:   jmp_buf jmpEnv;
michael@0:   jmp_buf retEnv;
michael@0: 
michael@0:   int flags;
michael@0: 
michael@0:   void *(*startupFunc)(void *arg);
michael@0:   void *startupArg;
michael@0: 
michael@0:   // The thread specific function to recreate the new thread. It's executed
michael@0:   // after the thread is recreated.
michael@0:   void (*recrFunc)(void *arg);
michael@0:   void *recrArg;
michael@0: 
michael@0:   TLSInfoList tlsInfo;
michael@0: 
michael@0:   pthread_mutex_t *reacquireMutex;
michael@0:   void *stk;
michael@0: 
michael@0:   pid_t origNativeThreadID;
michael@0:   pid_t recreatedNativeThreadID;
michael@0:   char nativeThreadName[NATIVE_THREAD_NAME_LENGTH];
michael@0: };
michael@0: 
michael@0: typedef struct thread_info thread_info_t;
michael@0: 
michael@0: static thread_info_t *sCurrentRecreatingThread = nullptr;
michael@0: 
michael@0: /**
michael@0:  * This function runs the custom recreation function registered when calling
michael@0:  * NuwaMarkCurrentThread() after thread stack is restored.
michael@0:  */
michael@0: static void
michael@0: RunCustomRecreation() {
michael@0:   thread_info_t *tinfo = sCurrentRecreatingThread;
michael@0:   if (tinfo->recrFunc != nullptr) {
michael@0:     tinfo->recrFunc(tinfo->recrArg);
michael@0:   }
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Every thread should be marked as either TINFO_FLAG_NUWA_SUPPORT or
michael@0:  * TINFO_FLAG_NUWA_SKIP, or it means a potential error.  We force
michael@0:  * Gecko code to mark every single thread to make sure there are no accidents
michael@0:  * when recreating threads with Nuwa.
michael@0:  *
michael@0:  * Threads marked as TINFO_FLAG_NUWA_SUPPORT can be checkpointed explicitly, by
michael@0:  * calling NuwaCheckpointCurrentThread(), or implicitly when they call into wrapped
michael@0:  * functions like pthread_mutex_lock(), epoll_wait(), etc.
michael@0:  * TINFO_FLAG_NUWA_EXPLICIT_CHECKPOINT denotes the explicitly checkpointed thread.
michael@0:  */
michael@0: #define TINFO_FLAG_NUWA_SUPPORT 0x1
michael@0: #define TINFO_FLAG_NUWA_SKIP 0x2
michael@0: #define TINFO_FLAG_NUWA_EXPLICIT_CHECKPOINT 0x4
michael@0: 
michael@0: typedef struct nuwa_construct {
michael@0:   void (*construct)(void *);
michael@0:   void *arg;
michael@0: } nuwa_construct_t;
michael@0: 
michael@0: static std::vector<nuwa_construct_t> sConstructors;
michael@0: static std::vector<nuwa_construct_t> sFinalConstructors;
michael@0: 
michael@0: typedef std::map<pthread_key_t, void (*)(void *)> TLSKeySet;
michael@0: static TLSKeySet sTLSKeys;
michael@0: 
michael@0: /**
michael@0:  * This mutex is used to block the running threads and freeze their contexts.
michael@0:  * PrepareNuwaProcess() is the first one to acquire the lock. Further attempts
michael@0:  * to acquire this mutex (in the freeze point macros) will block and freeze the
michael@0:  * calling thread.
michael@0:  */
michael@0: static pthread_mutex_t sThreadFreezeLock = PTHREAD_MUTEX_INITIALIZER;
michael@0: 
michael@0: static thread_info_t sMainThread;
michael@0: static LinkedList<thread_info_t> sAllThreads;
michael@0: static int sThreadCount = 0;
michael@0: static int sThreadFreezeCount = 0;
michael@0: /**
michael@0:  * This mutex protects the access to thread info:
michael@0:  * sAllThreads, sThreadCount, sThreadFreezeCount, sRecreateVIPCount.
michael@0:  */
michael@0: static pthread_mutex_t sThreadCountLock = PTHREAD_MUTEX_INITIALIZER;
michael@0: /**
michael@0:  * This condition variable lets MakeNuwaProcess() wait until all recreated
michael@0:  * threads are frozen.
michael@0:  */
michael@0: static pthread_cond_t sThreadChangeCond = PTHREAD_COND_INITIALIZER;
michael@0: 
michael@0: /**
michael@0:  * This mutex and condition variable is used to serialize the fork requests
michael@0:  * from the parent process.
michael@0:  */
michael@0: static pthread_mutex_t sForkLock = PTHREAD_MUTEX_INITIALIZER;
michael@0: static pthread_cond_t sForkWaitCond = PTHREAD_COND_INITIALIZER;
michael@0: 
michael@0: /**
michael@0:  * sForkWaitCondChanged will be reset to false on the IPC thread before
michael@0:  * and will be changed to true on the main thread to indicate that the condition
michael@0:  * that the IPC thread is waiting for has already changed.
michael@0:  */
michael@0: static bool sForkWaitCondChanged = false;
michael@0: 
michael@0: /**
michael@0:  * This mutex protects the access to sTLSKeys, which keeps track of existing
michael@0:  * TLS Keys.
michael@0:  */
michael@0: static pthread_mutex_t sTLSKeyLock = PTHREAD_MUTEX_INITIALIZER;
michael@0: static int sThreadSkipCount = 0;
michael@0: 
michael@0: static thread_info_t *
michael@0: GetThreadInfoInner(pthread_t threadID) {
michael@0:   for (thread_info_t *tinfo = sAllThreads.getFirst();
michael@0:        tinfo;
michael@0:        tinfo = tinfo->getNext()) {
michael@0:     if (pthread_equal(tinfo->origThreadID, threadID)) {
michael@0:       return tinfo;
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   return nullptr;
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Get thread info using the specified thread ID.
michael@0:  *
michael@0:  * @return thread_info_t which has threadID == specified threadID
michael@0:  */
michael@0: static thread_info_t *
michael@0: GetThreadInfo(pthread_t threadID) {
michael@0:   if (sIsNuwaProcess) {
michael@0:     REAL(pthread_mutex_lock)(&sThreadCountLock);
michael@0:   }
michael@0:   thread_info_t *tinfo = GetThreadInfoInner(threadID);
michael@0:   if (sIsNuwaProcess) {
michael@0:     pthread_mutex_unlock(&sThreadCountLock);
michael@0:   }
michael@0:   return tinfo;
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Get thread info using the specified native thread ID.
michael@0:  *
michael@0:  * @return thread_info_t with nativeThreadID == specified threadID
michael@0:  */
michael@0: static thread_info_t*
michael@0: GetThreadInfo(pid_t threadID) {
michael@0:   if (sIsNuwaProcess) {
michael@0:     REAL(pthread_mutex_lock)(&sThreadCountLock);
michael@0:   }
michael@0:   thread_info_t *thrinfo = nullptr;
michael@0:   for (thread_info_t *tinfo = sAllThreads.getFirst();
michael@0:        tinfo;
michael@0:        tinfo = tinfo->getNext()) {
michael@0:     if (tinfo->origNativeThreadID == threadID) {
michael@0:       thrinfo = tinfo;
michael@0:       break;
michael@0:     }
michael@0:   }
michael@0:   if (sIsNuwaProcess) {
michael@0:     pthread_mutex_unlock(&sThreadCountLock);
michael@0:   }
michael@0: 
michael@0:   return thrinfo;
michael@0: }
michael@0: 
michael@0: #if !defined(HAVE_THREAD_TLS_KEYWORD)
michael@0: /**
michael@0:  * Get thread info of the current thread.
michael@0:  *
michael@0:  * @return thread_info_t for the current thread.
michael@0:  */
michael@0: static thread_info_t *
michael@0: GetCurThreadInfo() {
michael@0:   pthread_t threadID = REAL(pthread_self)();
michael@0:   pthread_t thread_info_t::*threadIDptr =
michael@0:       (sIsNuwaProcess ?
michael@0:          &thread_info_t::origThreadID :
michael@0:          &thread_info_t::recreatedThreadID);
michael@0: 
michael@0:   REAL(pthread_mutex_lock)(&sThreadCountLock);
michael@0:   thread_info_t *tinfo;
michael@0:   for (tinfo = sAllThreads.getFirst();
michael@0:        tinfo;
michael@0:        tinfo = tinfo->getNext()) {
michael@0:     if (pthread_equal(tinfo->*threadIDptr, threadID)) {
michael@0:       break;
michael@0:     }
michael@0:   }
michael@0:   pthread_mutex_unlock(&sThreadCountLock);
michael@0:   return tinfo;
michael@0: }
michael@0: #define CUR_THREAD_INFO GetCurThreadInfo()
michael@0: #define SET_THREAD_INFO(x) /* Nothing to do. */
michael@0: #else
michael@0: // Is not nullptr only for threads created by pthread_create() in an Nuwa process.
michael@0: // It is always nullptr for the main thread.
michael@0: static __thread thread_info_t *sCurThreadInfo = nullptr;
michael@0: #define CUR_THREAD_INFO sCurThreadInfo
michael@0: #define SET_THREAD_INFO(x) do { sCurThreadInfo = (x); } while(0)
michael@0: #endif  // HAVE_THREAD_TLS_KEYWORD
michael@0: 
michael@0: /*
michael@0:  * Track all epoll fds and handling events.
michael@0:  */
michael@0: class EpollManager {
michael@0: public:
michael@0:   class EpollInfo {
michael@0:   public:
michael@0:     typedef struct epoll_event Events;
michael@0:     typedef std::map<int, Events> EpollEventsMap;
michael@0:     typedef EpollEventsMap::iterator iterator;
michael@0:     typedef EpollEventsMap::const_iterator const_iterator;
michael@0: 
michael@0:     EpollInfo(): mBackSize(0) {}
michael@0:     EpollInfo(int aBackSize): mBackSize(aBackSize) {}
michael@0:     EpollInfo(const EpollInfo &aOther): mEvents(aOther.mEvents)
michael@0:                                       , mBackSize(aOther.mBackSize) {
michael@0:     }
michael@0:     ~EpollInfo() {
michael@0:       mEvents.clear();
michael@0:     }
michael@0: 
michael@0:     void AddEvents(int aFd, Events &aEvents) {
michael@0:       std::pair<iterator, bool> pair =
michael@0:         mEvents.insert(std::make_pair(aFd, aEvents));
michael@0:       if (!pair.second) {
michael@0:         abort();
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     void RemoveEvents(int aFd) {
michael@0:       if (!mEvents.erase(aFd)) {
michael@0:         abort();
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     void ModifyEvents(int aFd, Events &aEvents) {
michael@0:       iterator it = mEvents.find(aFd);
michael@0:       if (it == mEvents.end()) {
michael@0:         abort();
michael@0:       }
michael@0:       it->second = aEvents;
michael@0:     }
michael@0: 
michael@0:     const Events &FindEvents(int aFd) const {
michael@0:       const_iterator it = mEvents.find(aFd);
michael@0:       if (it == mEvents.end()) {
michael@0:         abort();
michael@0:       }
michael@0:       return it->second;
michael@0:     }
michael@0: 
michael@0:     int Size() const { return mEvents.size(); }
michael@0: 
michael@0:     // Iterator with values of <fd, Events> pairs.
michael@0:     const_iterator begin() const { return mEvents.begin(); }
michael@0:     const_iterator end() const { return mEvents.end(); }
michael@0: 
michael@0:     int BackSize() const { return mBackSize; }
michael@0: 
michael@0:   private:
michael@0:     EpollEventsMap mEvents;
michael@0:     int mBackSize;
michael@0: 
michael@0:     friend class EpollManager;
michael@0:   };
michael@0: 
michael@0:   typedef std::map<int, EpollInfo> EpollInfoMap;
michael@0:   typedef EpollInfoMap::iterator iterator;
michael@0:   typedef EpollInfoMap::const_iterator const_iterator;
michael@0: 
michael@0: public:
michael@0:   void AddEpollInfo(int aEpollFd, int aBackSize) {
michael@0:     EpollInfo *oldinfo = FindEpollInfo(aEpollFd);
michael@0:     if (oldinfo != nullptr) {
michael@0:       abort();
michael@0:     }
michael@0:     mEpollFdsInfo[aEpollFd] = EpollInfo(aBackSize);
michael@0:   }
michael@0: 
michael@0:   EpollInfo *FindEpollInfo(int aEpollFd) {
michael@0:     iterator it = mEpollFdsInfo.find(aEpollFd);
michael@0:     if (it == mEpollFdsInfo.end()) {
michael@0:       return nullptr;
michael@0:     }
michael@0:     return &it->second;
michael@0:   }
michael@0: 
michael@0:   void RemoveEpollInfo(int aEpollFd) {
michael@0:     if (!mEpollFdsInfo.erase(aEpollFd)) {
michael@0:       abort();
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   int Size() const { return mEpollFdsInfo.size(); }
michael@0: 
michael@0:   // Iterator of <epollfd, EpollInfo> pairs.
michael@0:   const_iterator begin() const { return mEpollFdsInfo.begin(); }
michael@0:   const_iterator end() const { return mEpollFdsInfo.end(); }
michael@0: 
michael@0:   static EpollManager *Singleton() {
michael@0:     if (!sInstance) {
michael@0:       sInstance = new EpollManager();
michael@0:     }
michael@0:     return sInstance;
michael@0:   }
michael@0: 
michael@0:   static void Shutdown() {
michael@0:     if (!sInstance) {
michael@0:       abort();
michael@0:     }
michael@0: 
michael@0:     delete sInstance;
michael@0:     sInstance = nullptr;
michael@0:   }
michael@0: 
michael@0: private:
michael@0:   static EpollManager *sInstance;
michael@0:   ~EpollManager() {
michael@0:     mEpollFdsInfo.clear();
michael@0:   }
michael@0: 
michael@0:   EpollInfoMap mEpollFdsInfo;
michael@0: 
michael@0:   EpollManager() {}
michael@0: };
michael@0: 
michael@0: EpollManager* EpollManager::sInstance;
michael@0: 
michael@0: static thread_info_t *
michael@0: thread_info_new(void) {
michael@0:   /* link tinfo to sAllThreads */
michael@0:   thread_info_t *tinfo = new thread_info_t();
michael@0:   tinfo->flags = 0;
michael@0:   tinfo->recrFunc = nullptr;
michael@0:   tinfo->recrArg = nullptr;
michael@0:   tinfo->recreatedThreadID = 0;
michael@0:   tinfo->recreatedNativeThreadID = 0;
michael@0:   tinfo->reacquireMutex = nullptr;
michael@0:   tinfo->stk = malloc(NUWA_STACK_SIZE + getPageSize());
michael@0: 
michael@0:   // We use a smaller stack size. Add protection to stack overflow: mprotect()
michael@0:   // stack top (the page at the lowest address) so we crash instead of corrupt
michael@0:   // other content that is malloc()'d.
michael@0:   uintptr_t pageGuard = ceilToPage((uintptr_t)tinfo->stk);
michael@0:   mprotect((void*)pageGuard, getPageSize(), PROT_READ);
michael@0: 
michael@0:   pthread_attr_init(&tinfo->threadAttr);
michael@0: 
michael@0:   REAL(pthread_mutex_lock)(&sThreadCountLock);
michael@0:   // Insert to the tail.
michael@0:   sAllThreads.insertBack(tinfo);
michael@0: 
michael@0:   sThreadCount++;
michael@0:   pthread_cond_signal(&sThreadChangeCond);
michael@0:   pthread_mutex_unlock(&sThreadCountLock);
michael@0: 
michael@0:   return tinfo;
michael@0: }
michael@0: 
michael@0: static void
michael@0: thread_info_cleanup(void *arg) {
michael@0:   if (sNuwaForking) {
michael@0:     // We shouldn't have any thread exiting when we are forking a new process.
michael@0:     abort();
michael@0:   }
michael@0: 
michael@0:   thread_info_t *tinfo = (thread_info_t *)arg;
michael@0:   pthread_attr_destroy(&tinfo->threadAttr);
michael@0: 
michael@0:   REAL(pthread_mutex_lock)(&sThreadCountLock);
michael@0:   /* unlink tinfo from sAllThreads */
michael@0:   tinfo->remove();
michael@0: 
michael@0:   sThreadCount--;
michael@0:   pthread_cond_signal(&sThreadChangeCond);
michael@0:   pthread_mutex_unlock(&sThreadCountLock);
michael@0: 
michael@0:   free(tinfo->stk);
michael@0:   delete tinfo;
michael@0: }
michael@0: 
michael@0: static void *
michael@0: _thread_create_startup(void *arg) {
michael@0:   thread_info_t *tinfo = (thread_info_t *)arg;
michael@0:   void *r;
michael@0: 
michael@0:   // Save thread info; especially, stackaddr & stacksize.
michael@0:   // Reuse the stack in the new thread.
michael@0:   pthread_getattr_np(REAL(pthread_self)(), &tinfo->threadAttr);
michael@0: 
michael@0:   SET_THREAD_INFO(tinfo);
michael@0:   tinfo->origThreadID = REAL(pthread_self)();
michael@0:   tinfo->origNativeThreadID = gettid();
michael@0: 
michael@0:   pthread_cleanup_push(thread_info_cleanup, tinfo);
michael@0: 
michael@0:   r = tinfo->startupFunc(tinfo->startupArg);
michael@0: 
michael@0:   if (!sIsNuwaProcess) {
michael@0:     return r;
michael@0:   }
michael@0: 
michael@0:   pthread_cleanup_pop(1);
michael@0: 
michael@0:   return r;
michael@0: }
michael@0: 
michael@0: // reserve STACK_RESERVED_SZ * 4 bytes for thread_recreate_startup().
michael@0: #define STACK_RESERVED_SZ 64
michael@0: #define STACK_SENTINEL(v) ((v)[0])
michael@0: #define STACK_SENTINEL_VALUE(v) ((uint32_t)(v) ^ 0xdeadbeef)
michael@0: 
michael@0: static void *
michael@0: thread_create_startup(void *arg) {
michael@0:   /*
michael@0:    * Dark Art!! Never try to do the same unless you are ABSOLUTELY sure of
michael@0:    * what you are doing!
michael@0:    *
michael@0:    * This function is here for reserving stack space before calling
michael@0:    * _thread_create_startup().  see also thread_create_startup();
michael@0:    */
michael@0:   void *r;
michael@0:   volatile uint32_t reserved[STACK_RESERVED_SZ];
michael@0: 
michael@0:   // Reserve stack space.
michael@0:   STACK_SENTINEL(reserved) = STACK_SENTINEL_VALUE(reserved);
michael@0: 
michael@0:   r = _thread_create_startup(arg);
michael@0: 
michael@0:   // Check if the reservation is enough.
michael@0:   if (STACK_SENTINEL(reserved) != STACK_SENTINEL_VALUE(reserved)) {
michael@0:     abort();                    // Did not reserve enough stack space.
michael@0:   }
michael@0: 
michael@0:   thread_info_t *tinfo = CUR_THREAD_INFO;
michael@0:   if (!sIsNuwaProcess) {
michael@0:     longjmp(tinfo->retEnv, 1);
michael@0: 
michael@0:     // Never go here!
michael@0:     abort();
michael@0:   }
michael@0: 
michael@0:   return r;
michael@0: }
michael@0: 
michael@0: extern "C" MFBT_API int
michael@0: __wrap_pthread_create(pthread_t *thread,
michael@0:                       const pthread_attr_t *attr,
michael@0:                       void *(*start_routine) (void *),
michael@0:                       void *arg) {
michael@0:   if (!sIsNuwaProcess) {
michael@0:     return REAL(pthread_create)(thread, attr, start_routine, arg);
michael@0:   }
michael@0: 
michael@0:   thread_info_t *tinfo = thread_info_new();
michael@0:   tinfo->startupFunc = start_routine;
michael@0:   tinfo->startupArg = arg;
michael@0:   pthread_attr_setstack(&tinfo->threadAttr, tinfo->stk, NUWA_STACK_SIZE);
michael@0: 
michael@0:   int rv = REAL(pthread_create)(thread,
michael@0:                                 &tinfo->threadAttr,
michael@0:                                 thread_create_startup,
michael@0:                                 tinfo);
michael@0:   if (rv) {
michael@0:     thread_info_cleanup(tinfo);
michael@0:   } else {
michael@0:     tinfo->origThreadID = *thread;
michael@0:   }
michael@0: 
michael@0:   return rv;
michael@0: }
michael@0: 
michael@0: // TLS related
michael@0: 
michael@0: /**
michael@0:  * Iterates over the existing TLS keys and store the TLS data for the current
michael@0:  * thread in tinfo.
michael@0:  */
michael@0: static void
michael@0: SaveTLSInfo(thread_info_t *tinfo) {
michael@0:   REAL(pthread_mutex_lock)(&sTLSKeyLock);
michael@0:   tinfo->tlsInfo.clear();
michael@0:   for (TLSKeySet::const_iterator it = sTLSKeys.begin();
michael@0:        it != sTLSKeys.end();
michael@0:        it++) {
michael@0:     void *value = pthread_getspecific(it->first);
michael@0:     if (value == nullptr) {
michael@0:       continue;
michael@0:     }
michael@0: 
michael@0:     pthread_key_t key = it->first;
michael@0:     tinfo->tlsInfo.push_back(TLSInfoList::value_type(key, value));
michael@0:   }
michael@0:   pthread_mutex_unlock(&sTLSKeyLock);
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Restores the TLS data for the current thread from tinfo.
michael@0:  */
michael@0: static void
michael@0: RestoreTLSInfo(thread_info_t *tinfo) {
michael@0:   for (TLSInfoList::const_iterator it = tinfo->tlsInfo.begin();
michael@0:        it != tinfo->tlsInfo.end();
michael@0:        it++) {
michael@0:     pthread_key_t key = it->first;
michael@0:     const void *value = it->second;
michael@0:     if (pthread_setspecific(key, value)) {
michael@0:       abort();
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   SET_THREAD_INFO(tinfo);
michael@0:   tinfo->recreatedThreadID = REAL(pthread_self)();
michael@0:   tinfo->recreatedNativeThreadID = gettid();
michael@0: }
michael@0: 
michael@0: extern "C" MFBT_API int
michael@0: __wrap_pthread_key_create(pthread_key_t *key, void (*destructor)(void*)) {
michael@0:   int rv = REAL(pthread_key_create)(key, destructor);
michael@0:   if (rv != 0) {
michael@0:     return rv;
michael@0:   }
michael@0:   REAL(pthread_mutex_lock)(&sTLSKeyLock);
michael@0:   sTLSKeys.insert(TLSKeySet::value_type(*key, destructor));
michael@0:   pthread_mutex_unlock(&sTLSKeyLock);
michael@0:   return 0;
michael@0: }
michael@0: 
michael@0: extern "C" MFBT_API int
michael@0: __wrap_pthread_key_delete(pthread_key_t key) {
michael@0:   if (!sIsNuwaProcess) {
michael@0:     return REAL(pthread_key_delete)(key);
michael@0:   }
michael@0:   int rv = REAL(pthread_key_delete)(key);
michael@0:   if (rv != 0) {
michael@0:     return rv;
michael@0:   }
michael@0:   REAL(pthread_mutex_lock)(&sTLSKeyLock);
michael@0:   sTLSKeys.erase(key);
michael@0:   pthread_mutex_unlock(&sTLSKeyLock);
michael@0:   return 0;
michael@0: }
michael@0: 
michael@0: extern "C" MFBT_API pthread_t
michael@0: __wrap_pthread_self() {
michael@0:   thread_info_t *tinfo = CUR_THREAD_INFO;
michael@0:   if (tinfo) {
michael@0:     // For recreated thread, masquerade as the original thread in the Nuwa
michael@0:     // process.
michael@0:     return tinfo->origThreadID;
michael@0:   }
michael@0:   return REAL(pthread_self)();
michael@0: }
michael@0: 
michael@0: extern "C" MFBT_API int
michael@0: __wrap_pthread_join(pthread_t thread, void **retval) {
michael@0:   thread_info_t *tinfo = GetThreadInfo(thread);
michael@0:   if (tinfo == nullptr) {
michael@0:     return REAL(pthread_join)(thread, retval);
michael@0:   }
michael@0:   // pthread_join() need to use the real thread ID in the spawned process.
michael@0:   return REAL(pthread_join)(tinfo->recreatedThreadID, retval);
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * The following are used to synchronize between the main thread and the
michael@0:  * thread being recreated. The main thread will wait until the thread is woken
michael@0:  * up from the freeze points or the blocking intercepted functions and then
michael@0:  * proceed to recreate the next frozen thread.
michael@0:  *
michael@0:  * In thread recreation, the main thread recreates the frozen threads one by
michael@0:  * one. The recreated threads will be "gated" until the main thread "opens the
michael@0:  * gate" to let them run freely as if they were created from scratch. The VIP
michael@0:  * threads gets the chance to run first after their thread stacks are recreated
michael@0:  * (using longjmp()) so they can adjust their contexts to a valid, consistent
michael@0:  * state. The threads frozen waiting for pthread condition variables are VIP
michael@0:  * threads. After woken up they need to run first to make the associated mutex
michael@0:  * in a valid state to maintain the semantics of the intercepted function calls
michael@0:  * (like pthread_cond_wait()).
michael@0:  */
michael@0: 
michael@0: // Used to synchronize the main thread and the thread being recreated so that
michael@0: // only one thread is allowed to be recreated at a time.
michael@0: static pthread_mutex_t sRecreateWaitLock = PTHREAD_MUTEX_INITIALIZER;
michael@0: // Used to block recreated threads until the main thread "opens the gate".
michael@0: static pthread_mutex_t sRecreateGateLock = PTHREAD_MUTEX_INITIALIZER;
michael@0: // Used to block the main thread from "opening the gate" until all VIP threads
michael@0: // have been recreated.
michael@0: static pthread_mutex_t sRecreateVIPGateLock = PTHREAD_MUTEX_INITIALIZER;
michael@0: static pthread_cond_t sRecreateVIPCond = PTHREAD_COND_INITIALIZER;
michael@0: static int sRecreateVIPCount = 0;
michael@0: static int sRecreateGatePassed = 0;
michael@0: 
michael@0: /**
michael@0:  * Thread recreation macros.
michael@0:  *
michael@0:  * The following macros are used in the forked process to synchronize and
michael@0:  * control the progress of thread recreation.
michael@0:  *
michael@0:  * 1. RECREATE_START() is first called in the beginning of thread
michael@0:  *    recreation to set sRecreateWaitLock and sRecreateGateLock in locked
michael@0:  *    state.
michael@0:  * 2. For each frozen thread:
michael@0:  *    2.1. RECREATE_BEFORE() to set the thread being recreated.
michael@0:  *    2.2. thread_recreate() to recreate the frozen thread.
michael@0:  *    2.3. Main thread calls RECREATE_WAIT() to wait on sRecreateWaitLock until
michael@0:  *         the thread is recreated from the freeze point and calls
michael@0:  *         RECREATE_CONTINUE() to release sRecreateWaitLock.
michael@0:  *    2.3. Non-VIP threads are blocked on RECREATE_GATE(). VIP threads calls
michael@0:  *         RECREATE_PASS_VIP() to mark that a VIP thread is successfully
michael@0:  *         recreated and then is blocked by calling RECREATE_GATE_VIP().
michael@0:  * 3. RECREATE_WAIT_ALL_VIP() to wait until all VIP threads passed, that is,
michael@0:  *    VIP threads already has their contexts (mainly pthread mutex) in a valid
michael@0:  *    state.
michael@0:  * 4. RECREATE_OPEN_GATE() to unblock threads blocked by sRecreateGateLock.
michael@0:  * 5. RECREATE_FINISH() to complete thread recreation.
michael@0:  */
michael@0: #define RECREATE_START()                          \
michael@0:   do {                                            \
michael@0:     REAL(pthread_mutex_lock)(&sRecreateWaitLock); \
michael@0:     REAL(pthread_mutex_lock)(&sRecreateGateLock); \
michael@0:   } while(0)
michael@0: #define RECREATE_BEFORE(info) do { sCurrentRecreatingThread = info; } while(0)
michael@0: #define RECREATE_WAIT() REAL(pthread_mutex_lock)(&sRecreateWaitLock)
michael@0: #define RECREATE_CONTINUE() do {                \
michael@0:     RunCustomRecreation();                      \
michael@0:     pthread_mutex_unlock(&sRecreateWaitLock);   \
michael@0:   } while(0)
michael@0: #define RECREATE_FINISH() pthread_mutex_unlock(&sRecreateWaitLock)
michael@0: #define RECREATE_GATE()                           \
michael@0:   do {                                            \
michael@0:     REAL(pthread_mutex_lock)(&sRecreateGateLock); \
michael@0:     sRecreateGatePassed++;                        \
michael@0:     pthread_mutex_unlock(&sRecreateGateLock);     \
michael@0:   } while(0)
michael@0: #define RECREATE_OPEN_GATE() pthread_mutex_unlock(&sRecreateGateLock)
michael@0: #define RECREATE_GATE_VIP()                       \
michael@0:   do {                                            \
michael@0:     REAL(pthread_mutex_lock)(&sRecreateGateLock); \
michael@0:     pthread_mutex_unlock(&sRecreateGateLock);     \
michael@0:   } while(0)
michael@0: #define RECREATE_PASS_VIP()                          \
michael@0:   do {                                               \
michael@0:     REAL(pthread_mutex_lock)(&sRecreateVIPGateLock); \
michael@0:     sRecreateGatePassed++;                           \
michael@0:     pthread_cond_signal(&sRecreateVIPCond);          \
michael@0:     pthread_mutex_unlock(&sRecreateVIPGateLock);     \
michael@0:   } while(0)
michael@0: #define RECREATE_WAIT_ALL_VIP()                        \
michael@0:   do {                                                 \
michael@0:     REAL(pthread_mutex_lock)(&sRecreateVIPGateLock);   \
michael@0:     while(sRecreateGatePassed < sRecreateVIPCount) {   \
michael@0:       REAL(pthread_cond_wait)(&sRecreateVIPCond,       \
michael@0:                         &sRecreateVIPGateLock);        \
michael@0:     }                                                  \
michael@0:     pthread_mutex_unlock(&sRecreateVIPGateLock);       \
michael@0:   } while(0)
michael@0: 
michael@0: /**
michael@0:  * Thread freeze points. Note that the freeze points are implemented as macros
michael@0:  * so as not to garble the content of the stack after setjmp().
michael@0:  *
michael@0:  * In the nuwa process, when a thread supporting nuwa calls a wrapper
michael@0:  * function, freeze point 1 setjmp()s to save the state. We only allow the
michael@0:  * thread to be frozen in the wrapper functions. If thread freezing is not
michael@0:  * enabled yet, the wrapper functions act like their wrapped counterparts,
michael@0:  * except for the extra actions in the freeze points. If thread freezing is
michael@0:  * enabled, the thread will be frozen by calling one of the wrapper functions.
michael@0:  * The threads can be frozen in any of the following points:
michael@0:  *
michael@0:  * 1) Freeze point 1: this is the point where we setjmp() in the nuwa process
michael@0:  *    and longjmp() in the spawned process. If freezing is enabled, then the
michael@0:  *    current thread blocks by acquiring an already locked mutex,
michael@0:  *    sThreadFreezeLock.
michael@0:  * 2) The wrapped function: the function that might block waiting for some
michael@0:  *    resource or condition.
michael@0:  * 3) Freeze point 2: blocks the current thread by acquiring sThreadFreezeLock.
michael@0:  *    If freezing is not enabled then revert the counter change in freeze
michael@0:  *    point 1.
michael@0:  */
michael@0: #define THREAD_FREEZE_POINT1()                                 \
michael@0:   bool freezeCountChg = false;                                 \
michael@0:   bool recreated = false;                                      \
michael@0:   volatile bool freezePoint2 = false;                          \
michael@0:   thread_info_t *tinfo;                                        \
michael@0:   if (sIsNuwaProcess &&                                        \
michael@0:       (tinfo = CUR_THREAD_INFO) &&                             \
michael@0:       (tinfo->flags & TINFO_FLAG_NUWA_SUPPORT) &&              \
michael@0:       !(tinfo->flags & TINFO_FLAG_NUWA_EXPLICIT_CHECKPOINT)) { \
michael@0:     if (!setjmp(tinfo->jmpEnv)) {                              \
michael@0:       REAL(pthread_mutex_lock)(&sThreadCountLock);             \
michael@0:       SaveTLSInfo(tinfo);                                      \
michael@0:       sThreadFreezeCount++;                                    \
michael@0:       freezeCountChg = true;                                   \
michael@0:       pthread_cond_signal(&sThreadChangeCond);                 \
michael@0:       pthread_mutex_unlock(&sThreadCountLock);                 \
michael@0:                                                                \
michael@0:       if (sIsFreezing) {                                       \
michael@0:         REAL(pthread_mutex_lock)(&sThreadFreezeLock);          \
michael@0:         /* Never return from the pthread_mutex_lock() call. */ \
michael@0:         abort();                                               \
michael@0:       }                                                        \
michael@0:     } else {                                                   \
michael@0:       RECREATE_CONTINUE();                                     \
michael@0:       RECREATE_GATE();                                         \
michael@0:       freezeCountChg = false;                                  \
michael@0:       recreated = true;                                        \
michael@0:     }                                                          \
michael@0:   }
michael@0: 
michael@0: #define THREAD_FREEZE_POINT1_VIP()                             \
michael@0:   bool freezeCountChg = false;                                 \
michael@0:   bool recreated = false;                                      \
michael@0:   volatile bool freezePoint1 = false;                          \
michael@0:   volatile bool freezePoint2 = false;                          \
michael@0:   thread_info_t *tinfo;                                        \
michael@0:   if (sIsNuwaProcess &&                                        \
michael@0:       (tinfo = CUR_THREAD_INFO) &&                             \
michael@0:       (tinfo->flags & TINFO_FLAG_NUWA_SUPPORT) &&              \
michael@0:       !(tinfo->flags & TINFO_FLAG_NUWA_EXPLICIT_CHECKPOINT)) { \
michael@0:     if (!setjmp(tinfo->jmpEnv)) {                              \
michael@0:       REAL(pthread_mutex_lock)(&sThreadCountLock);             \
michael@0:       SaveTLSInfo(tinfo);                                      \
michael@0:       sThreadFreezeCount++;                                    \
michael@0:       sRecreateVIPCount++;                                     \
michael@0:       freezeCountChg = true;                                   \
michael@0:       pthread_cond_signal(&sThreadChangeCond);                 \
michael@0:       pthread_mutex_unlock(&sThreadCountLock);                 \
michael@0:                                                                \
michael@0:       if (sIsFreezing) {                                       \
michael@0:         freezePoint1 = true;                                   \
michael@0:         REAL(pthread_mutex_lock)(&sThreadFreezeLock);          \
michael@0:         /* Never return from the pthread_mutex_lock() call. */ \
michael@0:         abort();                                               \
michael@0:       }                                                        \
michael@0:     } else {                                                   \
michael@0:       freezeCountChg = false;                                  \
michael@0:       recreated = true;                                        \
michael@0:     }                                                          \
michael@0:   }
michael@0: 
michael@0: #define THREAD_FREEZE_POINT2()                               \
michael@0:   if (freezeCountChg) {                                      \
michael@0:     REAL(pthread_mutex_lock)(&sThreadCountLock);             \
michael@0:     if (sNuwaReady && sIsNuwaProcess) {                      \
michael@0:       pthread_mutex_unlock(&sThreadCountLock);               \
michael@0:       freezePoint2 = true;                                   \
michael@0:       REAL(pthread_mutex_lock)(&sThreadFreezeLock);          \
michael@0:       /* Never return from the pthread_mutex_lock() call. */ \
michael@0:       abort();                                               \
michael@0:     }                                                        \
michael@0:     sThreadFreezeCount--;                                    \
michael@0:     pthread_cond_signal(&sThreadChangeCond);                 \
michael@0:     pthread_mutex_unlock(&sThreadCountLock);                 \
michael@0:   }
michael@0: 
michael@0: #define THREAD_FREEZE_POINT2_VIP()                           \
michael@0:   if (freezeCountChg) {                                      \
michael@0:     REAL(pthread_mutex_lock)(&sThreadCountLock);             \
michael@0:     if (sNuwaReady && sIsNuwaProcess) {                      \
michael@0:       pthread_mutex_unlock(&sThreadCountLock);               \
michael@0:       freezePoint2 = true;                                   \
michael@0:       REAL(pthread_mutex_lock)(&sThreadFreezeLock);          \
michael@0:       /* Never return from the pthread_mutex_lock() call. */ \
michael@0:       abort();                                               \
michael@0:     }                                                        \
michael@0:     sThreadFreezeCount--;                                    \
michael@0:     sRecreateVIPCount--;                                     \
michael@0:     pthread_cond_signal(&sThreadChangeCond);                 \
michael@0:     pthread_mutex_unlock(&sThreadCountLock);                 \
michael@0:   }
michael@0: 
michael@0: /**
michael@0:  * Wrapping the blocking functions: epoll_wait(), poll(), pthread_mutex_lock(),
michael@0:  * pthread_cond_wait() and pthread_cond_timedwait():
michael@0:  *
michael@0:  * These functions are wrapped by the above freeze point macros. Once a new
michael@0:  * process is forked, the recreated thread will be blocked in one of the wrapper
michael@0:  * functions. When recreating the thread, we longjmp() to
michael@0:  * THREAD_FREEZE_POINT1() to recover the thread stack. Care must be taken to
michael@0:  * maintain the semantics of the wrapped function:
michael@0:  *
michael@0:  * - epoll_wait() and poll(): just retry the function.
michael@0:  * - pthread_mutex_lock(): don't lock if frozen at freeze point 2 (lock is
michael@0:  *   already acquired).
michael@0:  * - pthread_cond_wait() and pthread_cond_timedwait(): if the thread is frozen
michael@0:  *   waiting the condition variable, the mutex is already released, we need to
michael@0:  *   reacquire the mutex before calling the wrapped function again so the mutex
michael@0:  *   will be in a valid state.
michael@0:  */
michael@0: 
michael@0: extern "C" MFBT_API int
michael@0: __wrap_epoll_wait(int epfd,
michael@0:                   struct epoll_event *events,
michael@0:                   int maxevents,
michael@0:                   int timeout) {
michael@0:   int rv;
michael@0: 
michael@0:   THREAD_FREEZE_POINT1();
michael@0:   rv = REAL(epoll_wait)(epfd, events, maxevents, timeout);
michael@0:   THREAD_FREEZE_POINT2();
michael@0: 
michael@0:   return rv;
michael@0: }
michael@0: 
michael@0: extern "C" MFBT_API int
michael@0: __wrap_pthread_cond_wait(pthread_cond_t *cond,
michael@0:                          pthread_mutex_t *mtx) {
michael@0:   int rv = 0;
michael@0: 
michael@0:   THREAD_FREEZE_POINT1_VIP();
michael@0:   if (freezePoint2) {
michael@0:     RECREATE_CONTINUE();
michael@0:     RECREATE_PASS_VIP();
michael@0:     RECREATE_GATE_VIP();
michael@0:     return rv;
michael@0:   }
michael@0:   if (recreated && mtx) {
michael@0:     if (!freezePoint1 && pthread_mutex_trylock(mtx)) {
michael@0:       // The thread was frozen in pthread_cond_wait() after releasing mtx in the
michael@0:       // Nuwa process. In recreating this thread, We failed to reacquire mtx
michael@0:       // with the pthread_mutex_trylock() call, that is, mtx was acquired by
michael@0:       // another thread. Because of this, we need the main thread's help to
michael@0:       // reacquire mtx so that it will be in a valid state.
michael@0:       tinfo->reacquireMutex = mtx;
michael@0:     }
michael@0:     RECREATE_CONTINUE();
michael@0:     RECREATE_PASS_VIP();
michael@0:   }
michael@0:   rv = REAL(pthread_cond_wait)(cond, mtx);
michael@0:   if (recreated && mtx) {
michael@0:     // We still need to be gated as not to acquire another mutex associated with
michael@0:     // another VIP thread and interfere with it.
michael@0:     RECREATE_GATE_VIP();
michael@0:   }
michael@0:   THREAD_FREEZE_POINT2_VIP();
michael@0: 
michael@0:   return rv;
michael@0: }
michael@0: 
michael@0: extern "C" MFBT_API int
michael@0: __wrap_pthread_cond_timedwait(pthread_cond_t *cond,
michael@0:                               pthread_mutex_t *mtx,
michael@0:                               const struct timespec *abstime) {
michael@0:   int rv = 0;
michael@0: 
michael@0:   THREAD_FREEZE_POINT1_VIP();
michael@0:   if (freezePoint2) {
michael@0:     RECREATE_CONTINUE();
michael@0:     RECREATE_PASS_VIP();
michael@0:     RECREATE_GATE_VIP();
michael@0:     return rv;
michael@0:   }
michael@0:   if (recreated && mtx) {
michael@0:     if (!freezePoint1 && pthread_mutex_trylock(mtx)) {
michael@0:       tinfo->reacquireMutex = mtx;
michael@0:     }
michael@0:     RECREATE_CONTINUE();
michael@0:     RECREATE_PASS_VIP();
michael@0:   }
michael@0:   rv = REAL(pthread_cond_timedwait)(cond, mtx, abstime);
michael@0:   if (recreated && mtx) {
michael@0:     RECREATE_GATE_VIP();
michael@0:   }
michael@0:   THREAD_FREEZE_POINT2_VIP();
michael@0: 
michael@0:   return rv;
michael@0: }
michael@0: 
michael@0: extern "C" int __pthread_cond_timedwait(pthread_cond_t *cond,
michael@0:                                         pthread_mutex_t *mtx,
michael@0:                                         const struct timespec *abstime,
michael@0:                                         clockid_t clock);
michael@0: 
michael@0: extern "C" MFBT_API int
michael@0: __wrap___pthread_cond_timedwait(pthread_cond_t *cond,
michael@0:                                 pthread_mutex_t *mtx,
michael@0:                                 const struct timespec *abstime,
michael@0:                                 clockid_t clock) {
michael@0:   int rv = 0;
michael@0: 
michael@0:   THREAD_FREEZE_POINT1_VIP();
michael@0:   if (freezePoint2) {
michael@0:     RECREATE_CONTINUE();
michael@0:     RECREATE_PASS_VIP();
michael@0:     RECREATE_GATE_VIP();
michael@0:     return rv;
michael@0:   }
michael@0:   if (recreated && mtx) {
michael@0:     if (!freezePoint1 && pthread_mutex_trylock(mtx)) {
michael@0:       tinfo->reacquireMutex = mtx;
michael@0:     }
michael@0:     RECREATE_CONTINUE();
michael@0:     RECREATE_PASS_VIP();
michael@0:   }
michael@0:   rv = REAL(__pthread_cond_timedwait)(cond, mtx, abstime, clock);
michael@0:   if (recreated && mtx) {
michael@0:     RECREATE_GATE_VIP();
michael@0:   }
michael@0:   THREAD_FREEZE_POINT2_VIP();
michael@0: 
michael@0:   return rv;
michael@0: }
michael@0: 
michael@0: extern "C" MFBT_API int
michael@0: __wrap_pthread_mutex_lock(pthread_mutex_t *mtx) {
michael@0:   int rv = 0;
michael@0: 
michael@0:   THREAD_FREEZE_POINT1();
michael@0:   if (freezePoint2) {
michael@0:     return rv;
michael@0:   }
michael@0:   rv = REAL(pthread_mutex_lock)(mtx);
michael@0:   THREAD_FREEZE_POINT2();
michael@0: 
michael@0:   return rv;
michael@0: }
michael@0: 
michael@0: extern "C" MFBT_API int
michael@0: __wrap_poll(struct pollfd *fds, nfds_t nfds, int timeout) {
michael@0:   int rv;
michael@0: 
michael@0:   THREAD_FREEZE_POINT1();
michael@0:   rv = REAL(poll)(fds, nfds, timeout);
michael@0:   THREAD_FREEZE_POINT2();
michael@0: 
michael@0:   return rv;
michael@0: }
michael@0: 
michael@0: extern "C" MFBT_API int
michael@0: __wrap_epoll_create(int size) {
michael@0:   int epollfd = REAL(epoll_create)(size);
michael@0: 
michael@0:   if (!sIsNuwaProcess) {
michael@0:     return epollfd;
michael@0:   }
michael@0: 
michael@0:   if (epollfd >= 0) {
michael@0:     EpollManager::Singleton()->AddEpollInfo(epollfd, size);
michael@0:   }
michael@0: 
michael@0:   return epollfd;
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Wrapping the functions to create file descriptor pairs. In the child process
michael@0:  * FD pairs are created for intra-process signaling. The generation of FD pairs
michael@0:  * need to be tracked in the nuwa process so they can be recreated in the
michael@0:  * spawned process.
michael@0:  */
michael@0: struct FdPairInfo {
michael@0:   enum {
michael@0:     kPipe,
michael@0:     kSocketpair
michael@0:   } call;
michael@0: 
michael@0:   int FDs[2];
michael@0:   int flags;
michael@0:   int domain;
michael@0:   int type;
michael@0:   int protocol;
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * Protects the access to sSingalFds.
michael@0:  */
michael@0: static pthread_mutex_t  sSignalFdLock = PTHREAD_MUTEX_INITIALIZER;
michael@0: static std::vector<FdPairInfo> sSignalFds;
michael@0: 
michael@0: extern "C" MFBT_API int
michael@0: __wrap_socketpair(int domain, int type, int protocol, int sv[2])
michael@0: {
michael@0:   int rv = REAL(socketpair)(domain, type, protocol, sv);
michael@0: 
michael@0:   if (!sIsNuwaProcess || rv < 0) {
michael@0:     return rv;
michael@0:   }
michael@0: 
michael@0:   REAL(pthread_mutex_lock)(&sSignalFdLock);
michael@0:   FdPairInfo signalFd;
michael@0:   signalFd.call = FdPairInfo::kSocketpair;
michael@0:   signalFd.FDs[0] = sv[0];
michael@0:   signalFd.FDs[1] = sv[1];
michael@0:   signalFd.domain = domain;
michael@0:   signalFd.type = type;
michael@0:   signalFd.protocol = protocol;
michael@0: 
michael@0:   sSignalFds.push_back(signalFd);
michael@0:   pthread_mutex_unlock(&sSignalFdLock);
michael@0: 
michael@0:   return rv;
michael@0: }
michael@0: 
michael@0: extern "C" MFBT_API int
michael@0: __wrap_pipe2(int __pipedes[2], int flags)
michael@0: {
michael@0:   int rv = REAL(pipe2)(__pipedes, flags);
michael@0:   if (!sIsNuwaProcess || rv < 0) {
michael@0:     return rv;
michael@0:   }
michael@0: 
michael@0:   REAL(pthread_mutex_lock)(&sSignalFdLock);
michael@0:   FdPairInfo signalFd;
michael@0:   signalFd.call = FdPairInfo::kPipe;
michael@0:   signalFd.FDs[0] = __pipedes[0];
michael@0:   signalFd.FDs[1] = __pipedes[1];
michael@0:   signalFd.flags = flags;
michael@0:   sSignalFds.push_back(signalFd);
michael@0:   pthread_mutex_unlock(&sSignalFdLock);
michael@0:   return rv;
michael@0: }
michael@0: 
michael@0: extern "C" MFBT_API int
michael@0: __wrap_pipe(int __pipedes[2])
michael@0: {
michael@0:   return __wrap_pipe2(__pipedes, 0);
michael@0: }
michael@0: 
michael@0: static void
michael@0: DupeSingleFd(int newFd, int origFd)
michael@0: {
michael@0:   struct stat sb;
michael@0:   if (fstat(origFd, &sb)) {
michael@0:     // Maybe the original FD is closed.
michael@0:     return;
michael@0:   }
michael@0:   int fd = fcntl(origFd, F_GETFD);
michael@0:   int fl = fcntl(origFd, F_GETFL);
michael@0:   dup2(newFd, origFd);
michael@0:   fcntl(origFd, F_SETFD, fd);
michael@0:   fcntl(origFd, F_SETFL, fl);
michael@0:   REAL(close)(newFd);
michael@0: }
michael@0: 
michael@0: extern "C" MFBT_API void
michael@0: ReplaceSignalFds()
michael@0: {
michael@0:   for (std::vector<FdPairInfo>::iterator it = sSignalFds.begin();
michael@0:        it < sSignalFds.end(); ++it) {
michael@0:     int fds[2];
michael@0:     int rc = 0;
michael@0:     switch (it->call) {
michael@0:     case FdPairInfo::kPipe:
michael@0:       rc = REAL(pipe2)(fds, it->flags);
michael@0:       break;
michael@0:     case FdPairInfo::kSocketpair:
michael@0:       rc = REAL(socketpair)(it->domain, it->type, it->protocol, fds);
michael@0:       break;
michael@0:     default:
michael@0:       continue;
michael@0:     }
michael@0: 
michael@0:     if (rc == 0) {
michael@0:       DupeSingleFd(fds[0], it->FDs[0]);
michael@0:       DupeSingleFd(fds[1], it->FDs[1]);
michael@0:     }
michael@0:   }
michael@0: }
michael@0: 
michael@0: extern "C" MFBT_API int
michael@0: __wrap_epoll_ctl(int aEpollFd, int aOp, int aFd, struct epoll_event *aEvent) {
michael@0:   int rv = REAL(epoll_ctl)(aEpollFd, aOp, aFd, aEvent);
michael@0: 
michael@0:   if (!sIsNuwaProcess || rv == -1) {
michael@0:     return rv;
michael@0:   }
michael@0: 
michael@0:   EpollManager::EpollInfo *info =
michael@0:     EpollManager::Singleton()->FindEpollInfo(aEpollFd);
michael@0:   if (info == nullptr) {
michael@0:     abort();
michael@0:   }
michael@0: 
michael@0:   switch(aOp) {
michael@0:   case EPOLL_CTL_ADD:
michael@0:     info->AddEvents(aFd, *aEvent);
michael@0:     break;
michael@0: 
michael@0:   case EPOLL_CTL_MOD:
michael@0:     info->ModifyEvents(aFd, *aEvent);
michael@0:     break;
michael@0: 
michael@0:   case EPOLL_CTL_DEL:
michael@0:     info->RemoveEvents(aFd);
michael@0:     break;
michael@0: 
michael@0:   default:
michael@0:     abort();
michael@0:   }
michael@0: 
michael@0:   return rv;
michael@0: }
michael@0: 
michael@0: // XXX: thinker: Maybe, we should also track dup, dup2, and other functions.
michael@0: extern "C" MFBT_API int
michael@0: __wrap_close(int aFd) {
michael@0:   int rv = REAL(close)(aFd);
michael@0:   if (!sIsNuwaProcess || rv == -1) {
michael@0:     return rv;
michael@0:   }
michael@0: 
michael@0:   EpollManager::EpollInfo *info =
michael@0:     EpollManager::Singleton()->FindEpollInfo(aFd);
michael@0:   if (info) {
michael@0:     EpollManager::Singleton()->RemoveEpollInfo(aFd);
michael@0:   }
michael@0: 
michael@0:   return rv;
michael@0: }
michael@0: 
michael@0: extern "C" MFBT_API int
michael@0: __wrap_tgkill(pid_t tgid, pid_t tid, int signalno)
michael@0: {
michael@0:   if (sIsNuwaProcess) {
michael@0:     return tgkill(tgid, tid, signalno);
michael@0:   }
michael@0: 
michael@0:   if (tid == sMainThread.origNativeThreadID) {
michael@0:     return tgkill(tgid, sMainThread.recreatedNativeThreadID, signalno);
michael@0:   }
michael@0: 
michael@0:   thread_info_t *tinfo = (tid == sMainThread.origNativeThreadID ?
michael@0:       &sMainThread :
michael@0:       GetThreadInfo(tid));
michael@0:   if (!tinfo) {
michael@0:     return tgkill(tgid, tid, signalno);
michael@0:   }
michael@0: 
michael@0:   return tgkill(tgid, tinfo->recreatedNativeThreadID, signalno);
michael@0: }
michael@0: 
michael@0: static void *
michael@0: thread_recreate_startup(void *arg) {
michael@0:   /*
michael@0:    * Dark Art!! Never do the same unless you are ABSOLUTELY sure what you are
michael@0:    * doing!
michael@0:    *
michael@0:    * The stack space collapsed by this frame had been reserved by
michael@0:    * thread_create_startup().  And thread_create_startup() will
michael@0:    * return immediately after returning from real start routine, so
michael@0:    * all collapsed values does not affect the result.
michael@0:    *
michael@0:    * All outer frames of thread_create_startup() and
michael@0:    * thread_recreate_startup() are equivalent, so
michael@0:    * thread_create_startup() will return successfully.
michael@0:    */
michael@0:   thread_info_t *tinfo = (thread_info_t *)arg;
michael@0: 
michael@0:   prctl(PR_SET_NAME, (unsigned long)&tinfo->nativeThreadName, 0, 0, 0);
michael@0:   RestoreTLSInfo(tinfo);
michael@0: 
michael@0:   if (setjmp(tinfo->retEnv) != 0) {
michael@0:     return nullptr;
michael@0:   }
michael@0: 
michael@0:   // longjump() to recreate the stack on the new thread.
michael@0:   longjmp(tinfo->jmpEnv, 1);
michael@0: 
michael@0:   // Never go here!
michael@0:   abort();
michael@0: 
michael@0:   return nullptr;
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Recreate the context given by tinfo at a new thread.
michael@0:  */
michael@0: static void
michael@0: thread_recreate(thread_info_t *tinfo) {
michael@0:   pthread_t thread;
michael@0: 
michael@0:   // Note that the thread_recreate_startup() runs on the stack specified by
michael@0:   // tinfo.
michael@0:   pthread_create(&thread, &tinfo->threadAttr, thread_recreate_startup, tinfo);
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Recreate all threads in a process forked from an Nuwa process.
michael@0:  */
michael@0: static void
michael@0: RecreateThreads() {
michael@0:   sIsNuwaProcess = false;
michael@0:   sIsFreezing = false;
michael@0: 
michael@0:   sMainThread.recreatedThreadID = pthread_self();
michael@0:   sMainThread.recreatedNativeThreadID = gettid();
michael@0: 
michael@0:   // Run registered constructors.
michael@0:   for (std::vector<nuwa_construct_t>::iterator ctr = sConstructors.begin();
michael@0:        ctr != sConstructors.end();
michael@0:        ctr++) {
michael@0:     (*ctr).construct((*ctr).arg);
michael@0:   }
michael@0:   sConstructors.clear();
michael@0: 
michael@0:   REAL(pthread_mutex_lock)(&sThreadCountLock);
michael@0:   thread_info_t *tinfo = sAllThreads.getFirst();
michael@0:   pthread_mutex_unlock(&sThreadCountLock);
michael@0: 
michael@0:   RECREATE_START();
michael@0:   while (tinfo != nullptr) {
michael@0:     if (tinfo->flags & TINFO_FLAG_NUWA_SUPPORT) {
michael@0:       RECREATE_BEFORE(tinfo);
michael@0:       thread_recreate(tinfo);
michael@0:       RECREATE_WAIT();
michael@0:       if (tinfo->reacquireMutex) {
michael@0:         REAL(pthread_mutex_lock)(tinfo->reacquireMutex);
michael@0:       }
michael@0:     } else if(!(tinfo->flags & TINFO_FLAG_NUWA_SKIP)) {
michael@0:       // An unmarked thread is found other than the main thread.
michael@0: 
michael@0:       // All threads should be marked as one of SUPPORT or SKIP, or
michael@0:       // abort the process to make sure all threads in the Nuwa
michael@0:       // process are Nuwa-aware.
michael@0:       abort();
michael@0:     }
michael@0: 
michael@0:     tinfo = tinfo->getNext();
michael@0:   }
michael@0:   RECREATE_WAIT_ALL_VIP();
michael@0:   RECREATE_OPEN_GATE();
michael@0: 
michael@0:   RECREATE_FINISH();
michael@0: 
michael@0:   // Run registered final constructors.
michael@0:   for (std::vector<nuwa_construct_t>::iterator ctr = sFinalConstructors.begin();
michael@0:        ctr != sFinalConstructors.end();
michael@0:        ctr++) {
michael@0:     (*ctr).construct((*ctr).arg);
michael@0:   }
michael@0:   sFinalConstructors.clear();
michael@0: }
michael@0: 
michael@0: extern "C" {
michael@0: 
michael@0: /**
michael@0:  * Recreate all epoll fds and restore status; include all events.
michael@0:  */
michael@0: static void
michael@0: RecreateEpollFds() {
michael@0:   EpollManager *man = EpollManager::Singleton();
michael@0: 
michael@0:   for (EpollManager::const_iterator info_it = man->begin();
michael@0:        info_it != man->end();
michael@0:        info_it++) {
michael@0:     int epollfd = info_it->first;
michael@0:     const EpollManager::EpollInfo *info = &info_it->second;
michael@0: 
michael@0:     int fdflags = fcntl(epollfd, F_GETFD);
michael@0:     if (fdflags == -1) {
michael@0:       abort();
michael@0:     }
michael@0:     int fl = fcntl(epollfd, F_GETFL);
michael@0:     if (fl == -1) {
michael@0:       abort();
michael@0:     }
michael@0: 
michael@0:     int newepollfd = REAL(epoll_create)(info->BackSize());
michael@0:     if (newepollfd == -1) {
michael@0:       abort();
michael@0:     }
michael@0:     int rv = REAL(close)(epollfd);
michael@0:     if (rv == -1) {
michael@0:       abort();
michael@0:     }
michael@0:     rv = dup2(newepollfd, epollfd);
michael@0:     if (rv == -1) {
michael@0:       abort();
michael@0:     }
michael@0:     rv = REAL(close)(newepollfd);
michael@0:     if (rv == -1) {
michael@0:       abort();
michael@0:     }
michael@0: 
michael@0:     rv = fcntl(epollfd, F_SETFD, fdflags);
michael@0:     if (rv == -1) {
michael@0:       abort();
michael@0:     }
michael@0:     rv = fcntl(epollfd, F_SETFL, fl);
michael@0:     if (rv == -1) {
michael@0:       abort();
michael@0:     }
michael@0: 
michael@0:     for (EpollManager::EpollInfo::const_iterator events_it = info->begin();
michael@0:          events_it != info->end();
michael@0:          events_it++) {
michael@0:       int fd = events_it->first;
michael@0:       epoll_event events;
michael@0:       events = events_it->second;
michael@0:       rv = REAL(epoll_ctl)(epollfd, EPOLL_CTL_ADD, fd, &events);
michael@0:       if (rv == -1) {
michael@0:         abort();
michael@0:       }
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   // Shutdown EpollManager. It won't be needed in the spawned process.
michael@0:   EpollManager::Shutdown();
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Fix IPC to make it ready.
michael@0:  *
michael@0:  * Especially, fix ContentChild.
michael@0:  */
michael@0: static void
michael@0: ReplaceIPC(NuwaProtoFdInfo *aInfoList, int aInfoSize) {
michael@0:   int i;
michael@0:   int rv;
michael@0: 
michael@0:   for (i = 0; i < aInfoSize; i++) {
michael@0:     int fd = fcntl(aInfoList[i].originFd, F_GETFD);
michael@0:     if (fd == -1) {
michael@0:       abort();
michael@0:     }
michael@0: 
michael@0:     int fl = fcntl(aInfoList[i].originFd, F_GETFL);
michael@0:     if (fl == -1) {
michael@0:       abort();
michael@0:     }
michael@0: 
michael@0:     rv = dup2(aInfoList[i].newFds[NUWA_NEWFD_CHILD], aInfoList[i].originFd);
michael@0:     if (rv == -1) {
michael@0:       abort();
michael@0:     }
michael@0: 
michael@0:     rv = fcntl(aInfoList[i].originFd, F_SETFD, fd);
michael@0:     if (rv == -1) {
michael@0:       abort();
michael@0:     }
michael@0: 
michael@0:     rv = fcntl(aInfoList[i].originFd, F_SETFL, fl);
michael@0:     if (rv == -1) {
michael@0:       abort();
michael@0:     }
michael@0:   }
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Add a new content process at the chrome process.
michael@0:  */
michael@0: static void
michael@0: AddNewProcess(pid_t pid, NuwaProtoFdInfo *aInfoList, int aInfoSize) {
michael@0:   static bool (*AddNewIPCProcess)(pid_t, NuwaProtoFdInfo *, int) = nullptr;
michael@0: 
michael@0:   if (AddNewIPCProcess == nullptr) {
michael@0:     AddNewIPCProcess = (bool (*)(pid_t, NuwaProtoFdInfo *, int))
michael@0:       dlsym(RTLD_DEFAULT, "AddNewIPCProcess");
michael@0:   }
michael@0:   AddNewIPCProcess(pid, aInfoList, aInfoSize);
michael@0: }
michael@0: 
michael@0: static void
michael@0: PrepareProtoSockets(NuwaProtoFdInfo *aInfoList, int aInfoSize) {
michael@0:   int i;
michael@0:   int rv;
michael@0: 
michael@0:   for (i = 0; i < aInfoSize; i++) {
michael@0:     rv = REAL(socketpair)(PF_UNIX, SOCK_STREAM, 0, aInfoList[i].newFds);
michael@0:     if (rv == -1) {
michael@0:       abort();
michael@0:     }
michael@0:   }
michael@0: }
michael@0: 
michael@0: static void
michael@0: CloseAllProtoSockets(NuwaProtoFdInfo *aInfoList, int aInfoSize) {
michael@0:   int i;
michael@0: 
michael@0:   for (i = 0; i < aInfoSize; i++) {
michael@0:     REAL(close)(aInfoList[i].newFds[0]);
michael@0:     REAL(close)(aInfoList[i].newFds[1]);
michael@0:   }
michael@0: }
michael@0: 
michael@0: static void
michael@0: AfterForkHook()
michael@0: {
michael@0:   void (*AfterNuwaFork)();
michael@0: 
michael@0:   // This is defined in dom/ipc/ContentChild.cpp
michael@0:   AfterNuwaFork = (void (*)())
michael@0:     dlsym(RTLD_DEFAULT, "AfterNuwaFork");
michael@0:   AfterNuwaFork();
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Fork a new process that is ready for running IPC.
michael@0:  *
michael@0:  * @return the PID of the new process.
michael@0:  */
michael@0: static int
michael@0: ForkIPCProcess() {
michael@0:   int pid;
michael@0: 
michael@0:   REAL(pthread_mutex_lock)(&sForkLock);
michael@0: 
michael@0:   PrepareProtoSockets(sProtoFdInfos, sProtoFdInfosSize);
michael@0: 
michael@0:   sNuwaForking = true;
michael@0:   pid = fork();
michael@0:   sNuwaForking = false;
michael@0:   if (pid == -1) {
michael@0:     abort();
michael@0:   }
michael@0: 
michael@0:   if (pid > 0) {
michael@0:     // in the parent
michael@0:     AddNewProcess(pid, sProtoFdInfos, sProtoFdInfosSize);
michael@0:     CloseAllProtoSockets(sProtoFdInfos, sProtoFdInfosSize);
michael@0:   } else {
michael@0:     // in the child
michael@0:     if (getenv("MOZ_DEBUG_CHILD_PROCESS")) {
michael@0:       printf("\n\nNUWA CHILDCHILDCHILDCHILD\n  debug me @ %d\n\n", getpid());
michael@0:       sleep(30);
michael@0:     }
michael@0:     AfterForkHook();
michael@0:     ReplaceSignalFds();
michael@0:     ReplaceIPC(sProtoFdInfos, sProtoFdInfosSize);
michael@0:     RecreateEpollFds();
michael@0:     RecreateThreads();
michael@0:     CloseAllProtoSockets(sProtoFdInfos, sProtoFdInfosSize);
michael@0:   }
michael@0: 
michael@0:   sForkWaitCondChanged = true;
michael@0:   pthread_cond_signal(&sForkWaitCond);
michael@0:   pthread_mutex_unlock(&sForkLock);
michael@0: 
michael@0:   return pid;
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Prepare for spawning a new process. Called on the IPC thread.
michael@0:  */
michael@0: MFBT_API void
michael@0: NuwaSpawnPrepare() {
michael@0:   REAL(pthread_mutex_lock)(&sForkLock);
michael@0: 
michael@0:   sForkWaitCondChanged = false; // Will be modified on the main thread.
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Let IPC thread wait until fork action on the main thread has completed.
michael@0:  */
michael@0: MFBT_API void
michael@0: NuwaSpawnWait() {
michael@0:   while (!sForkWaitCondChanged) {
michael@0:     REAL(pthread_cond_wait)(&sForkWaitCond, &sForkLock);
michael@0:   }
michael@0:   pthread_mutex_unlock(&sForkLock);
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Spawn a new process. If not ready for spawn (still waiting for some threads
michael@0:  * to freeze), postpone the spawn request until ready.
michael@0:  *
michael@0:  * @return the pid of the new process, or 0 if not ready.
michael@0:  */
michael@0: MFBT_API pid_t
michael@0: NuwaSpawn() {
michael@0:   if (gettid() != getpid()) {
michael@0:     // Not the main thread.
michael@0:     abort();
michael@0:   }
michael@0: 
michael@0:   pid_t pid = 0;
michael@0: 
michael@0:   if (sNuwaReady) {
michael@0:     pid = ForkIPCProcess();
michael@0:   } else {
michael@0:     sNuwaPendingSpawn = true;
michael@0:   }
michael@0: 
michael@0:   return pid;
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Prepare to freeze the Nuwa-supporting threads.
michael@0:  */
michael@0: MFBT_API void
michael@0: PrepareNuwaProcess() {
michael@0:   sIsNuwaProcess = true;
michael@0:   // Explicitly ignore SIGCHLD so we don't have to call watpid() to reap
michael@0:   // dead child processes.
michael@0:   signal(SIGCHLD, SIG_IGN);
michael@0: 
michael@0:   // Make marked threads block in one freeze point.
michael@0:   REAL(pthread_mutex_lock)(&sThreadFreezeLock);
michael@0: 
michael@0:   // Populate sMainThread for mapping of tgkill.
michael@0:   sMainThread.origThreadID = pthread_self();
michael@0:   sMainThread.origNativeThreadID = gettid();
michael@0: }
michael@0: 
michael@0: // Make current process as a Nuwa process.
michael@0: MFBT_API void
michael@0: MakeNuwaProcess() {
michael@0:   void (*GetProtoFdInfos)(NuwaProtoFdInfo *, int, int *) = nullptr;
michael@0:   void (*OnNuwaProcessReady)() = nullptr;
michael@0:   sIsFreezing = true;
michael@0: 
michael@0:   REAL(pthread_mutex_lock)(&sThreadCountLock);
michael@0: 
michael@0:   // wait until all threads are frozen.
michael@0:   while ((sThreadFreezeCount + sThreadSkipCount) != sThreadCount) {
michael@0:     REAL(pthread_cond_wait)(&sThreadChangeCond, &sThreadCountLock);
michael@0:   }
michael@0: 
michael@0:   GetProtoFdInfos = (void (*)(NuwaProtoFdInfo *, int, int *))
michael@0:     dlsym(RTLD_DEFAULT, "GetProtoFdInfos");
michael@0:   GetProtoFdInfos(sProtoFdInfos, NUWA_TOPLEVEL_MAX, &sProtoFdInfosSize);
michael@0: 
michael@0:   sNuwaReady = true;
michael@0: 
michael@0:   pthread_mutex_unlock(&sThreadCountLock);
michael@0: 
michael@0:   OnNuwaProcessReady = (void (*)())dlsym(RTLD_DEFAULT, "OnNuwaProcessReady");
michael@0:   OnNuwaProcessReady();
michael@0: 
michael@0:   if (sNuwaPendingSpawn) {
michael@0:     sNuwaPendingSpawn = false;
michael@0:     NuwaSpawn();
michael@0:   }
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Mark the current thread as supporting Nuwa. The thread will be recreated in
michael@0:  * the spawned process.
michael@0:  */
michael@0: MFBT_API void
michael@0: NuwaMarkCurrentThread(void (*recreate)(void *), void *arg) {
michael@0:   if (!sIsNuwaProcess) {
michael@0:     return;
michael@0:   }
michael@0: 
michael@0:   thread_info_t *tinfo = CUR_THREAD_INFO;
michael@0:   if (tinfo == nullptr) {
michael@0:     abort();
michael@0:   }
michael@0: 
michael@0:   tinfo->flags |= TINFO_FLAG_NUWA_SUPPORT;
michael@0:   tinfo->recrFunc = recreate;
michael@0:   tinfo->recrArg = arg;
michael@0: 
michael@0:   // XXX Thread name might be set later than this call. If this is the case, we
michael@0:   // might need to delay getting the thread name.
michael@0:   prctl(PR_GET_NAME, (unsigned long)&tinfo->nativeThreadName, 0, 0, 0);
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Mark the current thread as not supporting Nuwa. Don't recreate this thread in
michael@0:  * the spawned process.
michael@0:  */
michael@0: MFBT_API void
michael@0: NuwaSkipCurrentThread() {
michael@0:   if (!sIsNuwaProcess) return;
michael@0: 
michael@0:   thread_info_t *tinfo = CUR_THREAD_INFO;
michael@0:   if (tinfo == nullptr) {
michael@0:     abort();
michael@0:   }
michael@0: 
michael@0:   if (!(tinfo->flags & TINFO_FLAG_NUWA_SKIP)) {
michael@0:     sThreadSkipCount++;
michael@0:   }
michael@0:   tinfo->flags |= TINFO_FLAG_NUWA_SKIP;
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Force to freeze the current thread.
michael@0:  *
michael@0:  * This method does not return in Nuwa process.  It returns for the
michael@0:  * recreated thread.
michael@0:  */
michael@0: MFBT_API void
michael@0: NuwaFreezeCurrentThread() {
michael@0:   thread_info_t *tinfo = CUR_THREAD_INFO;
michael@0:   if (sIsNuwaProcess &&
michael@0:       (tinfo = CUR_THREAD_INFO) &&
michael@0:       (tinfo->flags & TINFO_FLAG_NUWA_SUPPORT)) {
michael@0:     if (!setjmp(tinfo->jmpEnv)) {
michael@0:       REAL(pthread_mutex_lock)(&sThreadCountLock);
michael@0:       SaveTLSInfo(tinfo);
michael@0:       sThreadFreezeCount++;
michael@0:       pthread_cond_signal(&sThreadChangeCond);
michael@0:       pthread_mutex_unlock(&sThreadCountLock);
michael@0: 
michael@0:       REAL(pthread_mutex_lock)(&sThreadFreezeLock);
michael@0:     } else {
michael@0:       RECREATE_CONTINUE();
michael@0:       RECREATE_GATE();
michael@0:     }
michael@0:   }
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * The caller of NuwaCheckpointCurrentThread() is at the line it wishes to
michael@0:  * return after the thread is recreated.
michael@0:  *
michael@0:  * The checkpointed thread will restart at the calling line of
michael@0:  * NuwaCheckpointCurrentThread(). This macro returns true in the Nuwa process
michael@0:  * and false on the recreated thread in the forked process.
michael@0:  *
michael@0:  * NuwaCheckpointCurrentThread() is implemented as a macro so we can place the
michael@0:  * setjmp() call in the calling method without changing its stack pointer. This
michael@0:  * is essential for not corrupting the stack when the calling thread continues
michael@0:  * to request the main thread for forking a new process. The caller of
michael@0:  * NuwaCheckpointCurrentThread() should not return before the process forking
michael@0:  * finishes.
michael@0:  *
michael@0:  * @return true for Nuwa process, and false in the forked process.
michael@0:  */
michael@0: MFBT_API jmp_buf*
michael@0: NuwaCheckpointCurrentThread1() {
michael@0:   thread_info_t *tinfo = CUR_THREAD_INFO;
michael@0:   if (sIsNuwaProcess &&
michael@0:       (tinfo = CUR_THREAD_INFO) &&
michael@0:       (tinfo->flags & TINFO_FLAG_NUWA_SUPPORT)) {
michael@0:     return &tinfo->jmpEnv;
michael@0:   }
michael@0:   abort();
michael@0:   return nullptr;
michael@0: }
michael@0: 
michael@0: MFBT_API bool
michael@0: NuwaCheckpointCurrentThread2(int setjmpCond) {
michael@0:   thread_info_t *tinfo = CUR_THREAD_INFO;
michael@0:   if (setjmpCond == 0) {
michael@0:     REAL(pthread_mutex_lock)(&sThreadCountLock);
michael@0:     if (!(tinfo->flags & TINFO_FLAG_NUWA_EXPLICIT_CHECKPOINT)) {
michael@0:       tinfo->flags |= TINFO_FLAG_NUWA_EXPLICIT_CHECKPOINT;
michael@0:       SaveTLSInfo(tinfo);
michael@0:       sThreadFreezeCount++;
michael@0:     }
michael@0:     pthread_cond_signal(&sThreadChangeCond);
michael@0:     pthread_mutex_unlock(&sThreadCountLock);
michael@0:     return true;
michael@0:   }
michael@0:   RECREATE_CONTINUE();
michael@0:   RECREATE_GATE();
michael@0:   return false;               // Recreated thread.
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Register methods to be invoked before recreating threads in the spawned
michael@0:  * process.
michael@0:  */
michael@0: MFBT_API void
michael@0: NuwaAddConstructor(void (*construct)(void *), void *arg) {
michael@0:   nuwa_construct_t ctr;
michael@0:   ctr.construct = construct;
michael@0:   ctr.arg = arg;
michael@0:   sConstructors.push_back(ctr);
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Register methods to be invoked after recreating threads in the spawned
michael@0:  * process.
michael@0:  */
michael@0: MFBT_API void
michael@0: NuwaAddFinalConstructor(void (*construct)(void *), void *arg) {
michael@0:   nuwa_construct_t ctr;
michael@0:   ctr.construct = construct;
michael@0:   ctr.arg = arg;
michael@0:   sFinalConstructors.push_back(ctr);
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * @return if the current process is the nuwa process.
michael@0:  */
michael@0: MFBT_API bool
michael@0: IsNuwaProcess() {
michael@0:   return sIsNuwaProcess;
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * @return if the nuwa process is ready for spawning new processes.
michael@0:  */
michael@0: MFBT_API bool
michael@0: IsNuwaReady() {
michael@0:   return sNuwaReady;
michael@0: }
michael@0: 
michael@0: }      // extern "C"