SERVER-49842 Startup repair should rebuild indexes when validate index inconsistencies exceed the memory limit

jstests/disk/libs/wt_file_helper.js

@@ -55,7 +55,7 @@ let assertRepairSucceeds = function(dbpath, port, opts) {
             args.push("--" + a);
 
         if (opts[a].length > 0) {
-            args.push(a);
+            args.push(opts[a]);
         }
     }
     jsTestLog("Repairing the node");
@@ -221,8 +221,8 @@ let runWiredTigerTool = function(...args) {
  * Stops the given mongod, runs the truncate command on the given uri using the WiredTiger tool, and
  * starts mongod again on the same path.
  */
-let truncateUriAndRestartMongod = function(uri, conn) {
+let truncateUriAndRestartMongod = function(uri, conn, mongodOptions) {
     MongoRunner.stopMongod(conn, null, {skipValidation: true});
     runWiredTigerTool("-h", conn.dbpath, "truncate", uri);
-    return startMongodOnExistingPath(conn.dbpath, {});
+    return startMongodOnExistingPath(conn.dbpath, mongodOptions);
 };

jstests/disk/wt_repair_inconsistent_index.js

@@ -13,51 +13,103 @@ const collName = "test";
 const dbpath = MongoRunner.dataPath + baseName + "/";
 
 /**
- * Run the test by supplying additional paramters to MongoRunner.runMongod with 'mongodOptions'.
+ * Run the test by supplying additional parameters to MongoRunner.runMongod with 'mongodOptions'.
  */
 let runTest = function(mongodOptions) {
-    resetDbpath(dbpath);
     jsTestLog("Running test with args: " + tojson(mongodOptions));
 
     /**
-     * Test 1. Configure the skipIndexNewRecords failpoint, then insert documents into
-     * testColl, which will result in an index inconsistency. Run repair and verify
-     * that the index is rebuilt.
+     * Configure the skipIndexNewRecords failpoint, then insert documents into testColl, which will
+     * result in an index inconsistency. Run repair and verify that the index is fixed by validate
+     * without needing to be fully rebuilt.
      */
+    (function testInconsistentIndex() {
+        jsTestLog("Testing a repair on an inconsistent index");
+        resetDbpath(dbpath);
+        let mongod = startMongodOnExistingPath(dbpath, mongodOptions);
+        let testColl = mongod.getDB(baseName)[collName];
 
-    let mongod = startMongodOnExistingPath(dbpath, mongodOptions);
-    let testColl = mongod.getDB(baseName)[collName];
+        const doc = {a: 1};
+        assert.commandWorked(testColl.insert(doc));
 
-    const doc = {a: 1};
-    assert.commandWorked(testColl.insert(doc));
+        const indexName = "a_1";
+        assert.commandWorked(testColl.createIndex({a: 1}, {name: indexName}));
+        assertQueryUsesIndex(testColl, doc, indexName);
 
-    const indexName = "a_1";
-    assert.commandWorked(testColl.createIndex({a: 1}, {name: indexName}));
-    assertQueryUsesIndex(testColl, doc, indexName);
+        const testCollUri = getUriForColl(testColl);
+        const indexUri = getUriForIndex(testColl, indexName);
 
-    let testCollUri = getUriForColl(testColl);
-    let indexUri = getUriForIndex(testColl, indexName);
+        const db = mongod.getDB(baseName);
+        assert.commandWorked(
+            db.adminCommand({configureFailPoint: 'skipIndexNewRecords', mode: 'alwaysOn'}));
+        assert.commandWorked(testColl.insert({a: 2}));
 
-    let db = mongod.getDB(baseName);
-    assert.commandWorked(
-        db.adminCommand({configureFailPoint: 'skipIndexNewRecords', mode: 'alwaysOn'}));
-    assert.commandWorked(testColl.insert({a: 2}));
+        // Disable validation because it is expected to not pass due to index inconsistencies.
+        MongoRunner.stopMongod(mongod, null, {skipValidation: true});
 
-    // Disable validation because it is expected to not pass due to index inconsistencies.
-    MongoRunner.stopMongod(mongod, null, {skipValidation: true});
+        assertRepairSucceeds(dbpath, mongod.port, mongodOptions);
+        mongod = startMongodOnExistingPath(dbpath, mongodOptions);
+        testColl = mongod.getDB(baseName)[collName];
 
-    assertRepairSucceeds(dbpath, mongod.port, mongodOptions);
-    mongod = startMongodOnExistingPath(dbpath, mongodOptions);
-    testColl = mongod.getDB(baseName)[collName];
+        // Repair doesn't create new idents because validate repair mode fixed index
+        // inconsistencies.
+        assert.eq(indexUri, getUriForIndex(testColl, indexName));
 
-    // Repair doesn't create new idents because validate repair mode fixed index inconsistencies.
-    assert.eq(indexUri, getUriForIndex(testColl, indexName));
+        assertQueryUsesIndex(testColl, doc, indexName);
+        assert.eq(testCollUri, getUriForColl(testColl));
+        assert.eq(testColl.count(), 2);
 
-    assertQueryUsesIndex(testColl, doc, indexName);
-    assert.eq(testCollUri, getUriForColl(testColl));
-    assert.eq(testColl.count(), 2);
+        MongoRunner.stopMongod(mongod);
+    })();
 
-    MongoRunner.stopMongod(mongod);
+    /**
+     * Truncate an index and set the validate memory limit low enough so that validate repair mode
+     * cannot fix the index. This causes the index to be entirely rebuilt.
+     */
+    (function testInconsistentIndexPartialRepair() {
+        jsTestLog("Testing a partial repair on an inconsistent index");
+        resetDbpath(dbpath);
+        let mongod = startMongodOnExistingPath(dbpath, mongodOptions);
+        let testColl = mongod.getDB(baseName)[collName];
+
+        // Insert a document that is the size of the validate memory limit so that validate is
+        // unable to report and fix all inconsistencies during startup repair.
+        const bigDoc = {a: 'x'.repeat(1024 * 1024)};
+        assert.commandWorked(testColl.insert(bigDoc));
+
+        const smallDoc = {a: 1};
+        assert.commandWorked(testColl.insert(smallDoc));
+
+        const indexName = "a_1";
+        assert.commandWorked(testColl.createIndex({a: 1}, {name: indexName}));
+        assertQueryUsesIndex(testColl, {a: 1}, indexName);
+
+        const testCollUri = getUriForColl(testColl);
+        const indexUri = getUriForIndex(testColl, indexName);
+
+        // Remove all index entries and restart.
+        mongod = truncateUriAndRestartMongod(indexUri, mongod, mongodOptions);
+
+        // Disable validation because it is expected to fail due to index inconsistencies.
+        MongoRunner.stopMongod(mongod, null, {skipValidation: true});
+
+        // Impose a memory limit so that only one index key can be detected and repaired.
+        const options = mongodOptions;
+        options["setParameter"] = "maxValidateMemoryUsageMB=1";
+        assertRepairSucceeds(dbpath, mongod.port, options);
+        mongod = startMongodOnExistingPath(dbpath, options);
+        testColl = mongod.getDB(baseName)[collName];
+
+        // Repair should create a new ident because validate repair is unable to fix all index
+        // inconsistencies.
+        assert.neq(indexUri, getUriForIndex(testColl, indexName));
+
+        assertQueryUsesIndex(testColl, {a: 1}, indexName);
+        assert.eq(testCollUri, getUriForColl(testColl));
+        assert.eq(testColl.count(), 2);
+
+        MongoRunner.stopMongod(mongod);
+    })();
 };
 
 runTest({});

jstests/noPassthrough/validate_memory_limit.js

@@ -31,6 +31,13 @@ function checkValidate(errorPrefix, numMissingIndexEntries) {
     assert.eq(res.missingIndexEntries.length, numMissingIndexEntries);
 }
 
+function checkValidateRepair(expectRepair) {
+    const res = coll.validate({repair: true});
+    assert.commandWorked(res);
+    assert(!res.valid, printjson(res));
+    assert.eq(res.repaired, expectRepair, printjson(res));
+}
+
 const noneReportedPrefix =
     "Unable to report index entry inconsistencies due to memory limitations.";
 const notAllReportedPrefix =
@@ -43,10 +50,16 @@ assert.commandWorked(coll.insert({_id: indexKey}));
 corruptIndex();
 checkValidate(noneReportedPrefix, 0);
 
+// Can't repair successfully if there aren't any index inconsistencies reported.
+checkValidateRepair(false);
+
 // Insert a document with a small key so that validate reports one missing index entry.
 assert.commandWorked(coll.insert({_id: 1}));
 corruptIndex();
 checkValidate(notAllReportedPrefix, 1);
 
+// Repair, but incompletely if only some inconsistencies are reported.
+checkValidateRepair(true);
+
 MongoRunner.stopMongod(conn, null, {skipValidation: true});
 })();

src/mongo/db/catalog/index_consistency.cpp

@@ -447,6 +447,7 @@ bool IndexConsistency::limitMemoryUsageForSecondPhase(ValidateResults* result) {
     ss << "Not all index entry inconsistencies are reported due to memory limitations. "
        << memoryLimitMessage.str();
     result->errors.push_back(ss.str());
+    result->valid = false;
 
     return true;
 }