Clean up signal handling, fixing some potential hangs

db/db.cpp

@@ -313,10 +313,6 @@ namespace mongo {
             log() << "opLogging = " << opLogging << endl;
         _oplog.init();
 
-#if !defined(_WIN32)
-        assert( signal(SIGSEGV, segvhandler) != SIG_ERR );
-#endif
-
 #if !defined(_WIN32)
         pid_t pid = 0;
         pid = getpid();
@@ -334,8 +330,6 @@ namespace mongo {
         }
 #endif
       
-        setupSignals();
-
         repairDatabases();
 
         /* this is for security on certain platforms */
@@ -346,7 +340,6 @@ namespace mongo {
 
     int test2();
     void testClient();
-    void pipeSigHandler( int signal );
 
 } // namespace mongo
 
@@ -354,6 +347,8 @@ using namespace mongo;
 
 int main(int argc, char* argv[], char *envp[] )
 {
+    setupSignals();
+    
     dbExecCommand = argv[0];
     
     srand(curTimeMicros());
@@ -370,9 +365,6 @@ int main(int argc, char* argv[], char *envp[] )
 
     DEV out() << "warning: DEV mode enabled\n";
     
-#if !defined(_WIN32)
-    signal(SIGPIPE, pipeSigHandler);
-#endif
     UnitTest::runTests();
 
     if ( argc >= 2 ) {
@@ -577,27 +569,21 @@ namespace mongo {
         psignal( signal, "Signal Received : ");
     }
 
-    int segvs = 0;
-    void segvhandler(int x) {
-        if ( ++segvs > 1 ) {
-            signal(x, SIG_DFL);
-            if ( segvs == 2 ) {
-                out() << "\n\n\n got 2nd SIGSEGV" << endl;
-                sayDbContext();
-            }
-            return;
-        }
-        out() << "got SIGSEGV " << x << ", terminating :-(" << endl;
-        sayDbContext();
-//	closeAllSockets();
-//	MemoryMappedFile::closeAllFiles();
-//	flushOpLog();
-        dbexit(14);
+    void abruptQuit(int x) {
+        ostringstream oss;
+        oss << "Got signal: " << x << ", printing backtrace:" << endl;
+        printStackTrace( oss );
+        rawOut( oss.str() );
+        exit(14);
     }
 
-    void mysighandler(int x) {
-        signal(x, SIG_IGN);
-        out() << "got kill or ctrl c signal " << x << ", will terminate after current cmd ends" << endl;
+    sigset_t asyncSignals;
+    // The above signals will be processed by this thread only, in order to
+    // ensure the db and log mutexes aren't held.
+    void interruptThread() {
+        int x;
+        sigwait( &asyncSignals, &x );
+        log() << "got kill or ctrl c signal " << x << ", will terminate after current cmd ends" << endl;
         {
             dblock lk;
             log() << "now exiting" << endl;
@@ -606,8 +592,17 @@ namespace mongo {
     }
         
     void setupSignals() {
-        assert( signal(SIGINT, mysighandler) != SIG_ERR );
-        assert( signal(SIGTERM, mysighandler) != SIG_ERR );
+        assert( signal(SIGSEGV, abruptQuit) != SIG_ERR );
+        assert( signal(SIGFPE, abruptQuit) != SIG_ERR );
+        assert( signal(SIGABRT, abruptQuit) != SIG_ERR );
+        assert( signal(SIGBUS, abruptQuit) != SIG_ERR );
+        assert( signal(SIGPIPE, pipeSigHandler) != SIG_ERR );
+
+        sigemptyset( &asyncSignals );
+        sigaddset( &asyncSignals, SIGINT );
+        sigaddset( &asyncSignals, SIGTERM );
+        pthread_sigmask( SIG_SETMASK, &asyncSignals, 0 );
+        boost::thread it( interruptThread );
     }
 
 #else

db/instance.cpp

@@ -587,17 +587,34 @@ namespace mongo {
 
     DBDirectClient::AlwaysAuthorized DBDirectClient::Authorizer::always;
     
-    /* not using log() herein in case we are called from segvhandler and we were already locked */
-#undef exit
+    boost::mutex &exitMutex( *( new boost::mutex ) );
+    bool firstExit = true;
+    
+    /* not using log() herein in case we are already locked */
     void dbexit(int rc, const char *why) {        
-        out() << "dbexit: " << why << "; flushing op log and files" << endl;
+        {
+            boostlock lk( exitMutex );
+            if ( !firstExit ) {
+                stringstream ss;
+                ss << "dbexit: " << why << "; exiting immediately" << endl;
+                rawOut( ss.str() );
+                ::exit( rc );                
+            }
+            firstExit = false;
+        }
+            
+        stringstream ss;
+        ss << "dbexit: " << why << "; flushing op log and files" << endl;
+        rawOut( ss.str() );
         flushOpLog();
 
         /* must do this before unmapping mem or you may get a seg fault */
         closeAllSockets();
             
-        MemoryMappedFile::closeAllFiles();
-        out() << "dbexit: really exiting now" << endl;
+        stringstream ss2;
+        MemoryMappedFile::closeAllFiles( ss2 );
+        rawOut( ss2.str() );
+        rawOut( "dbexit: really exiting now\n" );
         ::exit(rc);
     }
 

stdafx.h

@@ -235,6 +235,11 @@ namespace mongo {
 
 #define null (0)
 
+    inline void rawOut( const string &s ) {
+        write( STDOUT_FILENO, s.c_str(), s.length() );
+        fsync( STDOUT_FILENO );        
+    }
+    
 } // namespace mongo
 
 #include <vector>

util/goodies.h

@@ -42,7 +42,7 @@ namespace mongo {
 namespace mongo {
 
     /* use "addr2line -CFe <exe>" to parse. */
-    inline void printStackTrace() {
+    inline void printStackTrace( ostream &o = cout ) {
         void *b[12];
         size_t size;
         char **strings;
@@ -52,10 +52,10 @@ namespace mongo {
         strings = backtrace_symbols(b, size);
 
         for (i = 0; i < size; i++)
-            cout << hex << b[i] << ' ';
-        cout << '\n';
+            o << hex << b[i] << ' ';
+        o << '\n';
         for (i = 0; i < size; i++)
-            cout << ' ' << strings[i] << '\n';
+            o << ' ' << strings[i] << '\n';
 
         free (strings);
     }

util/mmap.cpp

@@ -34,15 +34,15 @@ namespace mongo {
 
     /*static*/
     int closingAllFiles = 0;
-    void MemoryMappedFile::closeAllFiles() {
+    void MemoryMappedFile::closeAllFiles( stringstream &message ) {
         if ( closingAllFiles ) {
-            out() << "warning closingAllFiles=" << closingAllFiles << endl;
+            message << "warning closingAllFiles=" << closingAllFiles << endl;
             return;
         }
         ++closingAllFiles;
         for ( set<MemoryMappedFile*>::iterator i = mmfiles.begin(); i != mmfiles.end(); i++ )
             (*i)->close();
-        log() << "  closeAllFiles() finished" << endl;
+        message << "  closeAllFiles() finished" << endl;
         --closingAllFiles;
     }
 

util/mmap.h

@@ -22,7 +22,7 @@ namespace mongo {
 
     class MemoryMappedFile {
     public:
-        static void closeAllFiles();
+        static void closeAllFiles( stringstream &message );
         MemoryMappedFile();
         ~MemoryMappedFile(); /* closes the file if open */
         void close();