2011-08-04 17:17:45 +02:00
|
|
|
// More complete version of sharding/sync6.js
|
2012-11-05 18:11:19 +01:00
|
|
|
// Test that distributed lock forcing does not result in inconsistencies, using a
|
2011-08-04 17:17:45 +02:00
|
|
|
// fast timeout.
|
|
|
|
|
|
|
|
// Note that this test will always have random factors, since we can't control the
|
|
|
|
// thread scheduling.
|
|
|
|
|
2012-10-27 17:54:53 +02:00
|
|
|
var bitbucket = _isWindows() ? "NUL" : "/dev/null";
|
2012-10-29 01:14:48 +01:00
|
|
|
test = new SyncCCTest("sync6", {logpath: bitbucket, logappend: ""});
|
2011-08-04 17:17:45 +02:00
|
|
|
|
2012-11-05 18:11:19 +01:00
|
|
|
// Startup another process to handle our commands to the cluster, mostly so it's
|
2011-08-04 17:17:45 +02:00
|
|
|
// easier to read.
|
2012-11-05 18:11:19 +01:00
|
|
|
var commandConn = startMongodTest( 30000 + 4, "syncCommander", false, {}); //{ logpath : bitbucket } )//{verbose : ""} )
|
2013-09-26 03:59:42 +02:00
|
|
|
// { logpath : MongoRunner.dataDir + "/syncCommander/mongod.log" } );
|
2011-08-04 17:17:45 +02:00
|
|
|
|
|
|
|
// Up the log level for this test
|
2012-11-05 18:11:19 +01:00
|
|
|
commandConn.getDB( "admin" ).runCommand( { setParameter : 1, logLevel : 0 } );
|
2011-08-04 17:17:45 +02:00
|
|
|
|
|
|
|
// Have lots of threads, so use larger i
|
2012-11-05 18:11:19 +01:00
|
|
|
// Can't test too many, we get socket exceptions... possibly due to the
|
2011-08-04 17:17:45 +02:00
|
|
|
// javascript console.
|
2011-08-07 02:39:44 +02:00
|
|
|
// TODO: Figure out our max bounds here - use less threads now to avoid pinger starvation issues.
|
|
|
|
for ( var t = 0; t < 4; t++ ) {
|
|
|
|
for ( var i = 4; i < 5; i++ ) {
|
2011-08-04 17:17:45 +02:00
|
|
|
|
|
|
|
// Our force time is 6 seconds - slightly diff from sync6 to ensure exact time not important
|
|
|
|
var takeoverMS = 6000;
|
|
|
|
|
|
|
|
// Generate valid sleep and skew for this timeout
|
|
|
|
var threadSleepWithLock = takeoverMS / 2;
|
2012-11-05 18:11:19 +01:00
|
|
|
var configServerTimeSkew = [ 0, 0, 0 ];
|
2011-08-04 17:17:45 +02:00
|
|
|
for ( var h = 0; h < 3; h++ ) {
|
|
|
|
// Skew by 1/30th the takeover time either way, at max
|
2012-11-05 18:11:19 +01:00
|
|
|
configServerTimeSkew[h] = ( i + h ) % Math.floor( takeoverMS / 60 );
|
2011-08-04 17:17:45 +02:00
|
|
|
// Make skew pos or neg
|
|
|
|
configServerTimeSkew[h] *= ( ( i + h ) % 2 ) ? -1 : 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Build command
|
2012-11-05 18:11:19 +01:00
|
|
|
command = { _testDistLockWithSkew : 1 };
|
2011-08-04 17:17:45 +02:00
|
|
|
|
|
|
|
// Basic test parameters
|
|
|
|
command["lockName"] = "TimeSkewFailNewTest_lock_" + i;
|
2012-11-05 18:11:19 +01:00
|
|
|
command["host"] = test.url;
|
|
|
|
command["seed"] = i;
|
|
|
|
command["numThreads"] = ( i % 50 ) + 1;
|
2011-08-04 17:17:45 +02:00
|
|
|
|
|
|
|
// Critical values so we're sure of correct operation
|
2012-11-05 18:11:19 +01:00
|
|
|
command["takeoverMS"] = takeoverMS;
|
|
|
|
command["wait"] = 4 * takeoverMS; // so we must force the lock
|
|
|
|
command["skewHosts"] = configServerTimeSkew;
|
|
|
|
command["threadWait"] = threadSleepWithLock;
|
2011-08-04 17:17:45 +02:00
|
|
|
|
|
|
|
// Less critical test params
|
|
|
|
|
|
|
|
// 1/3 of threads will not release the lock
|
2012-11-05 18:11:19 +01:00
|
|
|
command["hangThreads"] = 3;
|
2011-08-04 17:17:45 +02:00
|
|
|
// Amount of time to wait before trying lock again
|
2012-11-05 18:11:19 +01:00
|
|
|
command["threadSleep"] = 1; // ( ( i + 1 ) * 100 ) % (takeoverMS / 4)
|
2011-08-04 17:17:45 +02:00
|
|
|
// Amount of total clock skew possible between locking threads (processes)
|
|
|
|
// This can be large now.
|
2012-11-05 18:11:19 +01:00
|
|
|
command["skewRange"] = ( command["takeoverMS"] * 3 ) * 60 * 1000;
|
2011-08-04 17:17:45 +02:00
|
|
|
|
|
|
|
// Double-check our sleep, host skew, and takeoverMS values again
|
|
|
|
|
|
|
|
// At maximum, our threads must sleep only half the lock timeout time.
|
2012-11-05 18:11:19 +01:00
|
|
|
assert( command["threadWait"] <= command["takeoverMS"] / 2 );
|
2011-08-04 17:17:45 +02:00
|
|
|
for ( var h = 0; h < command["skewHosts"].length; h++ ) {
|
|
|
|
// At maximum, our config server time skew needs to be less than 1/30th
|
|
|
|
// the total time skew (1/60th either way).
|
2012-11-05 18:11:19 +01:00
|
|
|
assert( Math.abs( command["skewHosts"][h] ) <= ( command["takeoverMS"] / 60 ) );
|
2011-08-04 17:17:45 +02:00
|
|
|
}
|
|
|
|
|
2012-11-05 18:11:19 +01:00
|
|
|
result = commandConn.getDB( "admin" ).runCommand( command );
|
|
|
|
printjson( result );
|
|
|
|
printjson( command );
|
2011-08-04 17:17:45 +02:00
|
|
|
assert( result.ok, "Skewed threads did not increment correctly." );
|
|
|
|
|
|
|
|
}
|
2011-08-07 02:39:44 +02:00
|
|
|
}
|
2011-08-04 17:17:45 +02:00
|
|
|
|
2012-11-05 18:11:19 +01:00
|
|
|
stopMongoProgram( 30004 );
|
2011-08-04 17:17:45 +02:00
|
|
|
test.stop();
|