maintenance mode for commands SERVER-3427

db/commands.h

@@ -95,6 +95,11 @@ namespace mongo {
         */
         virtual bool requiresAuth() { return true; }
 
+        /* Return true if a replica set secondary should go into "recovering"
+           (unreadable) state while running this command.
+         */
+        virtual bool maintenanceMode() const { return false; }
+
         /** @param webUI expose the command in the web ui as localhost:28017/<name>
             @param oldName an optional old, deprecated name for the command
         */

db/compact.cpp

@@ -263,6 +263,7 @@ namespace mongo {
         virtual LockType locktype() const { return NONE; }
         virtual bool adminOnly() const { return false; }
         virtual bool slaveOk() const { return true; }
+        virtual bool maintenanceMode() const { return true; }
         virtual bool logTheOp() { return false; }
         virtual void help( stringstream& help ) const {
             help << "compact collection\n"

db/dbcommands.cpp

@@ -349,6 +349,7 @@ namespace mongo {
         virtual bool slaveOk() const {
             return true;
         }
+        virtual bool maintenanceMode() const { return true; }
         virtual void help( stringstream& help ) const {
             help << "repair database.  also compacts. note: slow.";
         }
@@ -1792,6 +1793,10 @@ namespace mongo {
         if ( c->adminOnly() )
             log( 2 ) << "command: " << cmdObj << endl;
 
+        if (c->maintenanceMode() && theReplSet && theReplSet->isSecondary()) {
+            theReplSet->setMaintenanceMode(true);
+        }
+
         if ( c->locktype() == Command::NONE ) {
             // we also trust that this won't crash
             client.curop()->ensureStarted();
@@ -1799,6 +1804,11 @@ namespace mongo {
             int ok = c->run( dbname , cmdObj , queryOptions, errmsg , result , fromRepl );
             if ( ! ok )
                 result.append( "errmsg" , errmsg );
+
+            if (c->maintenanceMode() && theReplSet) {
+                theReplSet->setMaintenanceMode(false);
+            }
+
             return ok;
         }
 
@@ -1812,11 +1822,13 @@ namespace mongo {
         client.curop()->ensureStarted();
         Client::Context ctx( dbname , dbpath , &lk , c->requiresAuth() );
 
+        bool retval = true;
+
         try {
             string errmsg;
             if ( ! c->run(dbname, cmdObj, queryOptions, errmsg, result, fromRepl ) ) {
                 result.append( "errmsg" , errmsg );
-                return false;
+                retval = false;
             }
         }
         catch ( DBException& e ) {
@@ -1824,14 +1836,18 @@ namespace mongo {
             ss << "exception: " << e.what();
             result.append( "errmsg" , ss.str() );
             result.append( "code" , e.getCode() );
-            return false;
+            retval = false;
         }
 
-        if ( c->logTheOp() && ! fromRepl ) {
+        if ( retval && c->logTheOp() && ! fromRepl ) {
             logOp("c", cmdns, cmdObj);
         }
 
-        return true;
+        if (c->maintenanceMode() && theReplSet) {
+            theReplSet->setMaintenanceMode(false);
+        }
+
+        return retval;
     }
 
 

db/repl/rs.cpp

@@ -70,10 +70,27 @@ namespace mongo {
 
     void ReplSetImpl::changeState(MemberState s) { box.change(s, _self); }
 
+    void ReplSetImpl::setMaintenanceMode(const bool inc) {
+        lock lk(this);
+
+        if (inc) {
+            log() << "replSet going into maintenance mode (" << _maintenanceMode << " other tasks)" << rsLog;
+
+            _maintenanceMode++;
+            changeState(MemberState::RS_RECOVERING);
+        }
+        else {
+            _maintenanceMode--;
+            // no need to change state, syncTail will try to go live as a secondary soon
+
+            log() << "leaving maintenance mode (" << _maintenanceMode << " other tasks)" << rsLog;
+        }
+    }
+
     Member* ReplSetImpl::getMostElectable() {
         lock lk(this);
-        
-        Member *max = 0;        
+
+        Member *max = 0;
 
         for (set<unsigned>::iterator it = _electableSet.begin(); it != _electableSet.end(); it++) {
             const Member *temp = findById(*it);
@@ -298,8 +315,10 @@ namespace mongo {
         _currentSyncTarget(0),
         _hbmsgTime(0),
         _self(0),
+        _maintenanceMode(0),
         mgr( new Manager(this) ),
         ghost( new GhostSync(this) ) {
+
         _cfg = 0;
         memset(_hbmsg, 0, sizeof(_hbmsg));
         strcpy( _hbmsg , "initial startup" );

db/repl/rs.h

@@ -446,11 +446,20 @@ namespace mongo {
         List1<Member> _members; // all members of the set EXCEPT _self.
         ReplSetConfig::MemberCfg _config; // config of _self
         unsigned _id; // _id of _self
+
+        int _maintenanceMode; // if we should stay in recovering state
     public:
         // this is called from within a writelock in logOpRS
         unsigned selfId() const { return _id; }
         Manager *mgr;
         GhostSync *ghost;
+        /**
+         * This forces a secondary to go into recovering state and stay there
+         * until this is called again, passing in "false".  Multiple threads can
+         * call this and it will leave maintenance mode once all of the callers
+         * have called it again, passing in false.
+         */
+        void setMaintenanceMode(const bool inc);
     private:
         Member* head() const { return _members.head(); }
     public:

db/repl/rs_sync.cpp

@@ -188,6 +188,16 @@ namespace mongo {
     */
     bool ReplSetImpl::tryToGoLiveAsASecondary(OpTime& /*out*/ minvalid) {
         bool golive = false;
+
+        {
+            lock lk( this );
+
+            if (_maintenanceMode > 0) {
+                // we're not actually going live
+                return true;
+            }
+        }
+
         {
             readlock lk("local.replset.minvalid");
             BSONObj mv;

jstests/replsets/maintenance.js

@@ -0,0 +1,27 @@
+
+
+var replTest = new ReplSetTest( {name: 'unicomplex', nodes: 3} );
+var conns = replTest.startSet();
+replTest.initiate();
+
+// Make sure we have a master
+var master = replTest.getMaster();
+
+for (i=0;i<10000; i++) { master.getDB("bar").foo.insert({x:1,y:i,abc:123,str:"foo bar baz"}); }
+for (i=0;i<1000; i++) { master.getDB("bar").foo.update({y:i},{$push :{foo : "barrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr"}}); }
+
+replTest.awaitReplication();
+
+assert.soon(function() { return conns[2].getDB("admin").isMaster().secondary; });
+
+join = startParallelShell( "db.getSisterDB('bar').runCommand({compact : 'foo'});", replTest.ports[2] );
+
+print("check secondary goes to recovering");
+assert.soon(function() { return !conns[2].getDB("admin").isMaster().secondary; });
+
+print("joining");
+join();
+
+print("check secondary becomes a secondary again");
+assert.soon(function() { return conns[2].getDB("admin").isMaster().secondary; });
+