mongodb/jstests/replsets/read_operations_during_step_down.js

/*
 * Test that the read operations are not killed and their connections are also not
 * closed during step down.
 */
load('jstests/libs/parallelTester.js');
load("jstests/libs/curop_helpers.js");  // for waitForCurOpByFailPoint().
load("jstests/replsets/rslib.js");

(function() {

"use strict";

const testName = "readOpsDuringStepDown";
const dbName = "test";
const collName = "coll";

var rst = new ReplSetTest({name: testName, nodes: [{}, {rsConfig: {priority: 0}}]});
rst.startSet();
rst.initiate();

const primary = rst.getPrimary();
const primaryDB = primary.getDB(dbName);
const primaryAdmin = primary.getDB("admin");
const primaryColl = primaryDB[collName];
const collNss = primaryColl.getFullName();

TestData.dbName = dbName;
TestData.collName = collName;

jsTestLog("1. Do a document write");
assert.commandWorked(
        primaryColl.insert({_id: 0}, {"writeConcern": {"w": "majority"}}));
rst.awaitReplication();

// Open a cursor on primary.
const cursorIdToBeReadAfterStepDown =
    assert.commandWorked(primaryDB.runCommand({"find": collName, batchSize: 0})).cursor.id;

jsTestLog("2. Start blocking getMore cmd before step down");
const joinGetMoreThread = startParallelShell(() => {
    // Open another cursor on primary before step down.
    primaryDB = db.getSiblingDB(TestData.dbName);
    const cursorIdToBeReadDuringStepDown =
        assert.commandWorked(primaryDB.runCommand({"find": TestData.collName, batchSize: 0}))
            .cursor.id;

    // Enable the fail point for get more cmd.
    assert.commandWorked(db.adminCommand(
        {configureFailPoint: "waitAfterPinningCursorBeforeGetMoreBatch", mode: "alwaysOn"}));

    getMoreRes = assert.commandWorked(primaryDB.runCommand(
        {"getMore": cursorIdToBeReadDuringStepDown, collection: TestData.collName}));
    assert.docEq([{_id: 0}], getMoreRes.cursor.nextBatch);
}, primary.port);

// Wait for getmore cmd to reach the fail point.
waitForCurOpByFailPoint(primaryAdmin, collNss, "waitAfterPinningCursorBeforeGetMoreBatch");

jsTestLog("2. Start blocking find cmd before step down");
const joinFindThread = startParallelShell(() => {
    // Enable the fail point for find cmd.
    assert.commandWorked(
        db.adminCommand({configureFailPoint: "waitInFindBeforeMakingBatch", mode: "alwaysOn"}));

    var findRes = assert.commandWorked(
        db.getSiblingDB(TestData.dbName).runCommand({"find": TestData.collName}));
    assert.docEq([{_id: 0}], findRes.cursor.firstBatch);
}, primary.port);

// Wait for find cmd to reach the fail point.
waitForCurOpByFailPoint(primaryAdmin, collNss, "waitInFindBeforeMakingBatch");

jsTestLog("3. Make primary step down");
const joinStepDownThread = startParallelShell(() => {
    assert.commandWorked(db.adminCommand({"replSetStepDown": 100, "force": true}));
}, primary.port);

// Wait until the step down has started to kill user operations.
checkLog.contains(primary, "Starting to kill user operations");

// Enable "waitAfterCommandFinishesExecution" fail point to make sure the find and get more
// commands on database 'test' does not complete before step down.
setFailPoint(primaryAdmin,
             "waitAfterCommandFinishesExecution",
             {ns: collNss, commands: ["find", "getMore"]});

jsTestLog("4. Disable fail points");
clearFailPoint(primaryAdmin, "waitInFindBeforeMakingBatch");
clearFailPoint(primaryAdmin, "waitAfterPinningCursorBeforeGetMoreBatch");

// Wait until the primary transitioned to SECONDARY state.
joinStepDownThread();
rst.waitForState(primary, ReplSetTest.State.SECONDARY);

// We don't want to check if we have reached "waitAfterCommandFinishesExecution" fail point
// because we already know that the primary has stepped down successfully. This implies that
// the find and get more commands are still running even after the node stepped down.
clearFailPoint(primaryAdmin, "waitAfterCommandFinishesExecution");

// Wait for find & getmore thread to join.
joinGetMoreThread();
joinFindThread();

jsTestLog("5. Start get more cmd after step down");
var getMoreRes = assert.commandWorked(
    primaryDB.runCommand({"getMore": cursorIdToBeReadAfterStepDown, collection: collName}));
assert.docEq([{_id: 0}], getMoreRes.cursor.nextBatch);

// Validate that no operations got killed on step down and no network disconnection happened due
// to failed unacknowledged operations.
const replMetrics = assert.commandWorked(primaryAdmin.adminCommand({serverStatus: 1})).metrics.repl;
assert.eq(replMetrics.stateTransition.lastStateTransition, "stepDown");
assert.eq(replMetrics.stateTransition.userOperationsKilled, 0);
// Should account for find and getmore commands issued before step down.
assert.gte(replMetrics.stateTransition.userOperationsRunning, 2);
assert.eq(replMetrics.network.notPrimaryUnacknowledgedWrites, 0);

rst.stopSet();
})();