SERVER-70055 Make TTL monitor pick up config changes immediately

jstests/libs/ttl_util.js

@@ -13,8 +13,10 @@ const TTLUtil = class {
      * @param {DB} db   Database connection.
      * @param {boolean} waitForMajorityCommit   Only applies when 'db' is from a replica set, set to
      *     false to disable waiting for TTL deletes to become majority commited.
+     * @param {number} timeoutMillis   timeout in milliseconds for the TTL wait. Defaults to
+     *     whatever value is used for assert.soon.
      */
-    static waitForPass(db, waitForMajorityCommit = true) {
+    static waitForPass(db, waitForMajorityCommit = true, timeoutMillis = undefined) {
         // The 'ttl.passes' metric is incremented when the TTL monitor has finished a pass.
         // Depending on the timing of the pass, seeing an increment of this metric might not
         // necessarily imply the data we are expecting to be deleted has been seen, as the TTL pass
@@ -23,7 +25,7 @@ const TTLUtil = class {
         const ttlPasses = db.serverStatus().metrics.ttl.passes;
         assert.soon(function() {
             return db.serverStatus().metrics.ttl.passes > ttlPasses + 1;
-        });
+        }, "Expected 2 TTL passes but achieved less than 2 in the given time", timeoutMillis);
 
         // Readers using a "majority" read concern might expect TTL deletes to be visible after
         // waitForPass. TTL writes do not imply 'majority' nor 'j: true', and are made durable by

jstests/noPassthrough/ttl_changes_are_immediate.js

@@ -0,0 +1,28 @@
+// Ensure that changes to the TTL sleep time are reflected immediately.
+(function() {
+"use strict";
+load("jstests/libs/ttl_util.js");
+
+let runner = MongoRunner.runMongod({setParameter: "ttlMonitorSleepSecs=1000"});
+let db = runner.getDB("test");
+let coll = db.ttl_collection;
+coll.drop();
+
+// Create TTL index.
+assert.commandWorked(coll.createIndex({x: 1}, {expireAfterSeconds: 0}));
+
+// Insert expired docs.
+let now = new Date();
+assert.commandWorked(coll.insert({x: now}));
+assert.commandWorked(coll.insert({x: now}));
+
+// TTL monitor should now be waiting for 1000 seconds. Modify it to 1 second.
+assert.commandWorked(db.adminCommand({setParameter: 1, ttlMonitorSleepSecs: 1}));
+
+// TTL Monitor should now perform passes every second. A timeout here would mean we fail the test.
+TTLUtil.waitForPass(coll.getDB(), true, 20 * 1000);
+
+assert.eq(coll.count(), 0, "We should get 0 documents after TTL monitor run");
+
+MongoRunner.stopMongod(runner);
+})();

src/mongo/db/ttl.cpp

@@ -308,6 +308,10 @@ CounterMetric ttlCollSubpassesIncreasedPriority("ttl.collSubpassesIncreasedPrior
 
 using MtabType = TenantMigrationAccessBlocker::BlockerType;
 
+TTLMonitor::TTLMonitor()
+    : BackgroundJob(false /* selfDelete */),
+      _ttlMonitorSleepSecs(Seconds{ttlMonitorSleepSecs.load()}) {}
+
 TTLMonitor* TTLMonitor::get(ServiceContext* serviceCtx) {
     return getTTLMonitor(serviceCtx).get();
 }
@@ -323,19 +327,48 @@ void TTLMonitor::set(ServiceContext* serviceCtx, std::unique_ptr<TTLMonitor> mon
     ttlMonitor = std::move(monitor);
 }
 
+Status TTLMonitor::onUpdateTTLMonitorSleepSeconds(int newSleepSeconds) {
+    if (auto client = Client::getCurrent()) {
+        if (auto ttlMonitor = TTLMonitor::get(client->getServiceContext())) {
+            ttlMonitor->updateSleepSeconds(Seconds{newSleepSeconds});
+        }
+    }
+    return Status::OK();
+}
+
+void TTLMonitor::updateSleepSeconds(Seconds newSeconds) {
+    {
+        stdx::lock_guard lk(_stateMutex);
+        _ttlMonitorSleepSecs = newSeconds;
+    }
+    _notificationCV.notify_all();
+}
+
 void TTLMonitor::run() {
     ThreadClient tc(name(), getGlobalServiceContext());
     AuthorizationSession::get(cc())->grantInternalAuthorization(&cc());
 
     while (true) {
         {
-            // Wait until either ttlMonitorSleepSecs passes or a shutdown is requested.
-            auto deadline = Date_t::now() + Seconds(ttlMonitorSleepSecs.load());
+            auto startTime = Date_t::now();
+            // Wait until either ttlMonitorSleepSecs passes, a shutdown is requested, or the
+            // sleeping time has changed.
             stdx::unique_lock<Latch> lk(_stateMutex);
+            auto deadline = startTime + _ttlMonitorSleepSecs;
 
             MONGO_IDLE_THREAD_BLOCK;
-            _shuttingDownCV.wait_until(
-                lk, deadline.toSystemTimePoint(), [&] { return _shuttingDown; });
+            while (Date_t::now() <= deadline && !_shuttingDown) {
+                _notificationCV.wait_until(lk, deadline.toSystemTimePoint());
+                // Recompute the deadline in case the sleep time has changed since we started.
+                auto newDeadline = startTime + _ttlMonitorSleepSecs;
+                if (deadline != newDeadline) {
+                    LOGV2_INFO(7005501,
+                               "TTL sleep deadline has changed",
+                               "oldDeadline"_attr = deadline,
+                               "newDeadline"_attr = newDeadline);
+                    deadline = newDeadline;
+                }
+            }
 
             if (_shuttingDown) {
                 return;
@@ -374,7 +407,7 @@ void TTLMonitor::shutdown() {
     {
         stdx::lock_guard<Latch> lk(_stateMutex);
         _shuttingDown = true;
-        _shuttingDownCV.notify_one();
+        _notificationCV.notify_all();
     }
     wait();
     LOGV2(3684101, "Finished shutting down TTL collection monitor thread");

src/mongo/db/ttl.h

@@ -49,12 +49,14 @@ void shutdownTTLMonitor(ServiceContext* serviceContext);
 
 class TTLMonitor : public BackgroundJob {
 public:
-    explicit TTLMonitor() : BackgroundJob(false /* selfDelete */) {}
+    TTLMonitor();
 
     static TTLMonitor* get(ServiceContext* serviceCtx);
 
     static void set(ServiceContext* serviceCtx, std::unique_ptr<TTLMonitor> monitor);
 
+    static Status onUpdateTTLMonitorSleepSeconds(int newSleepSeconds);
+
     std::string name() const {
         return "TTLMonitor";
     }
@@ -71,6 +73,8 @@ public:
      */
     void onStepUp(OperationContext* opCtx);
 
+    void updateSleepSeconds(Seconds newSeconds);
+
     long long getTTLPasses_forTest();
     long long getTTLSubPasses_forTest();
 
@@ -154,11 +158,15 @@ private:
     // Protects the state below.
     mutable Mutex _stateMutex = MONGO_MAKE_LATCH("TTLMonitorStateMutex");
 
-    // Signaled to wake up the thread, if the thread is waiting. The thread will check whether
-    // _shuttingDown is set and stop accordingly.
-    mutable stdx::condition_variable _shuttingDownCV;
+    // Signaled to wake up the thread, if the thread is waiting. This condition variable is used to
+    // notify the thread of either:
+    // * The server is shutting down.
+    // * The ttlMonitorSleepSecs variable has changed.
+    // If the server is shutting down the monitor will stop.
+    mutable stdx::condition_variable _notificationCV;
 
     bool _shuttingDown = false;
+    Seconds _ttlMonitorSleepSecs;
 };
 
 }  // namespace mongo

src/mongo/db/ttl.idl

@@ -27,7 +27,8 @@
 
 global:
     cpp_namespace: mongo
-
+    cpp_includes:
+        - "mongo/db/ttl.h"
 server_parameters:
     ttlMonitorEnabled:
         description: "Enable the TTL monitor."
@@ -41,6 +42,7 @@ server_parameters:
         set_at: [ startup, runtime ]
         cpp_vartype: AtomicWord<int>
         cpp_varname: ttlMonitorSleepSecs
+        on_update: "TTLMonitor::onUpdateTTLMonitorSleepSeconds"
         default: 60
         validator:
             gt: 0