From f80d887a780449efb3a4425a0b7a1dd9510476d9 Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 20 May 2009 10:54:23 -0400 Subject: [PATCH 01/16] BUG SERVER-58 fix embedded array matching --- db/matcher.cpp | 5 +++-- dbtests/querytests.cpp | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/db/matcher.cpp b/db/matcher.cpp index 3c70fb0f76e..4a4175e62a6 100644 --- a/db/matcher.cpp +++ b/db/matcher.cpp @@ -386,6 +386,7 @@ namespace mongo { compareOp - Equality, LT, GT, etc. deep - out param. set to true/false if we scanned an array isArr - + nextArr - true if an array has already been found Special forms: @@ -424,7 +425,7 @@ namespace mongo { BSONElement z = ai.next(); if ( z.type() == Object ) { BSONObj eo = z.embeddedObject(); - int cmp = matchesDotted(fieldName, toMatch, eo, compareOp, deep, false, nextArr); + int cmp = matchesDotted(fieldName, toMatch, eo, compareOp, deep, false, true); if ( cmp > 0 ) { if ( deep ) *deep = true; return 1; @@ -446,7 +447,7 @@ namespace mongo { return -1; BSONObj eo = se.embeddedObject(); - return matchesDotted(p+1, toMatch, eo, compareOp, deep, se.type() == Array, true); + return matchesDotted(p+1, toMatch, eo, compareOp, deep, se.type() == Array, nextArr); } else { e = obj.getField(fieldName); } diff --git a/dbtests/querytests.cpp b/dbtests/querytests.cpp index 46b3d56a363..1e13193bef9 100644 --- a/dbtests/querytests.cpp +++ b/dbtests/querytests.cpp @@ -623,6 +623,22 @@ namespace QueryTests { ASSERT_EQUALS( 1U, client().count( ns, fromjson( "{i:{$in:['a']}}" ) ) ); } }; + + class EmbeddedArray : public ClientBase { + public: + ~EmbeddedArray() { + client().dropCollection( "unittests.querytests.EmbeddedArray" ); + } + void run() { + const char *ns = "unittests.querytests.EmbeddedArray"; + client().insert( ns, fromjson( "{foo:{bar:['spam']}}" ) ); + client().insert( ns, fromjson( "{foo:{bar:['spam','eggs']}}" ) ); + client().insert( ns, fromjson( "{bar:['spam']}" ) ); + client().insert( ns, fromjson( "{bar:['spam','eggs']}" ) ); + ASSERT_EQUALS( 2U, client().count( ns, BSON( "bar" << "spam" ) ) ); + ASSERT_EQUALS( 2U, client().count( ns, BSON( "foo.bar" << "spam" ) ) ); + } + }; class All : public Suite { public: @@ -657,6 +673,7 @@ namespace QueryTests { add< MinMax >(); add< DirectLocking >(); add< FastCountIn >(); + add< EmbeddedArray >(); } }; From 1d091ef8b3cf8063b7e0a111f5ed92eb7536a6de Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 20 May 2009 10:55:03 -0400 Subject: [PATCH 02/16] fix compiler warning MINOR --- shell/utils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell/utils.cpp b/shell/utils.cpp index 9483321ff26..afc70940351 100644 --- a/shell/utils.cpp +++ b/shell/utils.cpp @@ -136,7 +136,7 @@ namespace mongo { } sort( ports.begin(), ports.end() ); - for( int i = 1; i < ports.size(); ++i ) + for( unsigned i = 1; i < ports.size(); ++i ) massert( "duplicate ports allocated", ports[ i - 1 ] != ports[ i ] ); BSONObjBuilder b; b.append( "", ports ); From 6516e72042cd7e6ff88717b8a9bb5991f223c6c1 Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 20 May 2009 11:28:55 -0400 Subject: [PATCH 03/16] add option to output to file --- tools/export.cpp | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tools/export.cpp b/tools/export.cpp index 119f0be6d18..34f8b1da044 100644 --- a/tools/export.cpp +++ b/tools/export.cpp @@ -39,12 +39,18 @@ public: ("query,q" , po::value() , " query filter" ) ("fields,f" , po::value() , " comma seperated list of field names e.g. -f=name,age " ) ("csv","export to csv instead of json") + ("out,o", po::value(), "output file; if not specified, stdout is used") ; } int run(){ const string ns = getNS(); const bool csv = hasParam( "csv" ); + ostream *outPtr = &cout; + string outfile = getParam( "out" ); + if ( hasParam( "out" ) ) + outPtr = new ofstream( outfile.c_str() ); + ostream &out = *outPtr; BSONObj * fieldsToReturn = 0; BSONObj realFieldsToReturn; @@ -79,10 +85,10 @@ public: if ( csv ){ for ( vector::iterator i=fields.begin(); i != fields.end(); i++ ){ if ( i != fields.begin() ) - cout << ","; - cout << *i; + out << ","; + out << *i; } - cout << endl; + out << endl; } while ( cursor->more() ) { @@ -90,15 +96,15 @@ public: if ( csv ){ for ( vector::iterator i=fields.begin(); i != fields.end(); i++ ){ if ( i != fields.begin() ) - cout << ","; + out << ","; const BSONElement & e = obj[i->c_str()]; if ( ! e.eoo() ) - cout << e.jsonString( TenGen , false ); + out << e.jsonString( TenGen , false ); } - cout << endl; + out << endl; } else { - cout << obj.jsonString() << endl; + out << obj.jsonString() << endl; } } From 04d6ddd42bbd0b3a1497b9e989f486e12d3edf42 Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 20 May 2009 11:29:21 -0400 Subject: [PATCH 04/16] handle more non-server programs --- shell/utils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell/utils.cpp b/shell/utils.cpp index afc70940351..50e6785f666 100644 --- a/shell/utils.cpp +++ b/shell/utils.cpp @@ -217,7 +217,7 @@ namespace mongo { } argv_[ args.nFields() ] = 0; - if ( program == "mongo" ) + if ( program != "mongod" && program != "mongos" ) port_ = 0; else assert( port_ > 0 ); From b403a0c0f3b34e1e6f942c3a311972592afdcf31 Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 20 May 2009 11:29:40 -0400 Subject: [PATCH 05/16] add smokeTool --- SConstruct | 1 + jstests/tool/tool1.js | 36 +++++++++++++++++++++++++++++++++ mongo.xcodeproj/project.pbxproj | 10 +++++++++ 3 files changed, 47 insertions(+) create mode 100644 jstests/tool/tool1.js diff --git a/SConstruct b/SConstruct index 28d87ffd2b5..76b361a16a8 100644 --- a/SConstruct +++ b/SConstruct @@ -828,6 +828,7 @@ if not onlyServer and not noshell: addSmoketest( "smokeSharding", [ "mongo", "mongod", "mongos" ], [ jsDirTestSpec( "sharding" ) ] ) addSmoketest( "smokeJsPerf", [ "mongo" ], [ mongo[0].abspath + " " + jsSpec( [ "perf", "*.js" ] ) ] ) addSmoketest( "smokeQuota", [ "mongo" ], runShellTest ) + addSmoketest( "smokeTool", [ "mongo" ], [ jsDirTestSpec( "tool" ) ] ) mongodForTests = None mongodForTestsPort = "27017" diff --git a/jstests/tool/tool1.js b/jstests/tool/tool1.js new file mode 100644 index 00000000000..ba044f58fc0 --- /dev/null +++ b/jstests/tool/tool1.js @@ -0,0 +1,36 @@ +// mongo tool tests, very basic to start with + +baseName = "jstests_tool_tool1"; +dbPath = "/data/db/" + baseName; +externalPath = "/data/db/" + baseName + "/external" +externalFile = externalPath + "/export.json" + + +port = allocatePorts( 1 )[ 0 ]; + +m = startMongod( "--port", port, "--dbpath", dbPath, "--nohttpinterface" ); +resetDbpath( externalPath ); +c = m.getDB( baseName ).getCollection( baseName ); +c.save( { a: 1 } ); + +startMongoProgramNoConnect( "mongodump", "--host", "127.0.0.1:" + port, "--out", externalPath ); + +sleep( 3000 ); + +c.drop(); + +startMongoProgramNoConnect( "mongorestore", "--host", "127.0.0.1:" + port, "--dir", externalPath ); + +assert.soon( function() { return c.findOne() && 1 == c.findOne().a; } ); + +resetDbpath( externalPath ); + +startMongoProgramNoConnect( "mongoexport", "--host", "127.0.0.1:" + port, "-d", baseName, "-c", baseName, "--out", externalFile ); + +sleep( 3000 ); + +c.drop(); + +startMongoProgramNoConnect( "mongoimportjson", "--host", "127.0.0.1:" + port, "-d", baseName, "-c", baseName, "--file", externalFile ); + +assert.soon( function() { return c.findOne() && 1 == c.findOne().a; } ); diff --git a/mongo.xcodeproj/project.pbxproj b/mongo.xcodeproj/project.pbxproj index 3adc984980e..f9128a4073d 100644 --- a/mongo.xcodeproj/project.pbxproj +++ b/mongo.xcodeproj/project.pbxproj @@ -54,6 +54,7 @@ 933E22110F4327B2000209E3 /* perftest.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = perftest.cpp; sourceTree = ""; }; 933E22120F4327B2000209E3 /* perftest.o */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.objfile"; path = perftest.o; sourceTree = ""; }; 933EF9C70FC3434000C4B294 /* quota1.js */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.javascript; path = quota1.js; sourceTree = ""; }; + 9340BBEA0FC4521C00656DE0 /* tool1.js */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.javascript; path = tool1.js; sourceTree = ""; }; 9342232B0EF16D4F00608550 /* connpool.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = connpool.cpp; sourceTree = ""; }; 9342232C0EF16D4F00608550 /* connpool.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = connpool.h; sourceTree = ""; }; 9342232D0EF16D4F00608550 /* dbclient.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = dbclient.cpp; sourceTree = ""; }; @@ -378,6 +379,14 @@ path = quota; sourceTree = ""; }; + 9340BBE90FC4521C00656DE0 /* tool */ = { + isa = PBXGroup; + children = ( + 9340BBEA0FC4521C00656DE0 /* tool1.js */, + ); + path = tool; + sourceTree = ""; + }; 9342232A0EF16D4F00608550 /* client */ = { isa = PBXGroup; children = ( @@ -588,6 +597,7 @@ 93A8D1D10F37544800C92B85 /* jstests */ = { isa = PBXGroup; children = ( + 9340BBE90FC4521C00656DE0 /* tool */, 933EF9C60FC3434000C4B294 /* quota */, 93DB95170FC1D3D50047F950 /* capped4.js */, 93DB94A40FC1BFA30047F950 /* capped3.js */, From 226885b97c3960616af62076b72f1160b43ba7e2 Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 20 May 2009 11:33:19 -0400 Subject: [PATCH 06/16] add smokeTool to smokeAll MINOR --- SConstruct | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SConstruct b/SConstruct index 76b361a16a8..029010758c7 100644 --- a/SConstruct +++ b/SConstruct @@ -882,7 +882,7 @@ def addMongodReqTargets( env, target, source ): testEnv.Alias( "addMongodReqTargets", [], [addMongodReqTargets] ) testEnv.AlwaysBuild( "addMongodReqTargets" ) -testEnv.Alias( "smokeAll", [ "smoke", "mongosTest", "smokeClone", "smokeRepl", "addMongodReqTargets", "smokeDisk", "smokeSharding" ] ) +testEnv.Alias( "smokeAll", [ "smoke", "mongosTest", "smokeClone", "smokeRepl", "addMongodReqTargets", "smokeDisk", "smokeSharding", "smokeTool" ] ) testEnv.AlwaysBuild( "smokeAll" ) def addMongodReqNoJsTargets( env, target, source ): From da7a9b3f71a151b17665d7033370adb5507bf3fd Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 20 May 2009 11:39:12 -0400 Subject: [PATCH 07/16] make tools capable of running standalone, without a mongod instance --- SConstruct | 2 +- db/db.cpp | 9 --------- db/instance.cpp | 9 +++++++++ db/instance.h | 3 +++ dbtests/dbtests.cpp | 7 +------ tools/Tool.cpp | 40 ++++++++++++++++++++-------------------- tools/Tool.h | 9 +++++++-- tools/dump.cpp | 7 ++++--- tools/export.cpp | 2 +- tools/files.cpp | 2 +- tools/importJSON.cpp | 8 ++++---- tools/restore.cpp | 2 +- 12 files changed, 52 insertions(+), 48 deletions(-) diff --git a/SConstruct b/SConstruct index 28d87ffd2b5..f408591b96b 100644 --- a/SConstruct +++ b/SConstruct @@ -663,7 +663,7 @@ mongod = env.Program( "mongod" , commonFiles + coreDbFiles + serverOnlyFiles + [ Default( mongod ) # tools -allToolFiles = allClientFiles + [ "tools/Tool.cpp" ] +allToolFiles = commonFiles + coreDbFiles + serverOnlyFiles + [ "client/gridfs.cpp", "tools/Tool.cpp" ] env.Program( "mongodump" , allToolFiles + [ "tools/dump.cpp" ] ) env.Program( "mongorestore" , allToolFiles + [ "tools/restore.cpp" ] ) diff --git a/db/db.cpp b/db/db.cpp index 3d51a465993..58c7896afa5 100644 --- a/db/db.cpp +++ b/db/db.cpp @@ -328,15 +328,6 @@ namespace mongo { Timer startupSrandTimer; - void acquirePathLock() { -#if !defined(_WIN32) && !defined(__sunos__) - string name = ( boost::filesystem::path( dbpath ) / "mongod.lock" ).native_file_string(); - lockFile = open( name.c_str(), O_RDONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO ); - massert( "Unable to create / open lock file for dbpath: " + name, lockFile > 0 ); - massert( "Unable to acquire lock for dbpath: " + name, flock( lockFile, LOCK_EX | LOCK_NB ) == 0 ); -#endif - } - void _initAndListen(int listenPort, const char *appserverLoc = null) { #if !defined(_WIN32) diff --git a/db/instance.cpp b/db/instance.cpp index 8f7141f5d5d..b3ac8e06150 100644 --- a/db/instance.cpp +++ b/db/instance.cpp @@ -698,4 +698,13 @@ namespace mongo { ::exit(rc); } + void acquirePathLock() { +#if !defined(_WIN32) && !defined(__sunos__) + string name = ( boost::filesystem::path( dbpath ) / "mongod.lock" ).native_file_string(); + lockFile = open( name.c_str(), O_RDONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO ); + massert( "Unable to create / open lock file for dbpath: " + name, lockFile > 0 ); + massert( "Unable to acquire lock for dbpath: " + name, flock( lockFile, LOCK_EX | LOCK_NB ) == 0 ); +#endif + } + } // namespace mongo diff --git a/db/instance.h b/db/instance.h index b8920011f6a..75211dfb90f 100644 --- a/db/instance.h +++ b/db/instance.h @@ -144,5 +144,8 @@ namespace mongo { string oldName_; }; }; + + extern int lockFile; + void acquirePathLock(); } // namespace mongo diff --git a/dbtests/dbtests.cpp b/dbtests/dbtests.cpp index f22d5ec563a..c9e2b596694 100644 --- a/dbtests/dbtests.cpp +++ b/dbtests/dbtests.cpp @@ -89,12 +89,7 @@ int main( int argc, char** argv ) { dbpathSpec += "/"; dbpath = dbpathSpec.c_str(); -#if !defined(_WIN32) && !defined(__sunos__) - string name = ( boost::filesystem::path( dbpath ) / "mongod.lock" ).native_file_string(); - int lockFile = open( name.c_str(), O_RDONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO ); - massert( "Unable to create / open lock file for dbpath: " + name, lockFile > 0 ); - massert( "Unable to acquire lock for dbpath: " + name, flock( lockFile, LOCK_EX | LOCK_NB ) == 0 ); -#endif + acquirePathLock(); srand( seed ); printGitVersion(); diff --git a/tools/Tool.cpp b/tools/Tool.cpp index 3f273c86417..8aec59632ce 100644 --- a/tools/Tool.cpp +++ b/tools/Tool.cpp @@ -6,21 +6,13 @@ #include -namespace mongo { - DBClientBase *createDirectClient() { - cout << "no direct client available" << endl; - assert( false ); - return 0; - } -} // namespace mongo - using namespace std; using namespace mongo; namespace po = boost::program_options; mongo::Tool::Tool( string name , string defaultDB , string defaultCollection ) : - _name( name ) , _db( defaultDB ) , _coll( defaultCollection ){ + _name( name ) , _db( defaultDB ) , _coll( defaultCollection ), _useDirect() { _options = new po::options_description( name + " options" ); _options->add_options() @@ -28,6 +20,7 @@ mongo::Tool::Tool( string name , string defaultDB , string defaultCollection ) : ("host,h",po::value(), "mongo host to connect to" ) ("db,d",po::value(), "database to use" ) ("collection,c",po::value(), "collection to use (some commands)" ) + ("dbpath",po::value(), "directly access mongod data files in this path, instead of connecting to a mongod instance" ) ; } @@ -53,19 +46,26 @@ int mongo::Tool::main( int argc , char ** argv ){ printExtraHelp( cerr ); return 0; } - - const char * host = "127.0.0.1"; - if ( _params.count( "host" ) ) - host = _params["host"].as().c_str(); - - string errmsg; - if ( ! _conn.connect( host , errmsg ) ){ - cerr << "couldn't connect to [" << host << "] " << errmsg << endl; - return -1; + + if ( !hasParam( "dbpath" ) ) { + const char * host = "127.0.0.1"; + if ( _params.count( "host" ) ) + host = _params["host"].as().c_str(); + + string errmsg; + if ( ! _conn.connect( host , errmsg ) ){ + cerr << "couldn't connect to [" << host << "] " << errmsg << endl; + return -1; + } + + cerr << "connected to: " << host << endl; + } else { + _useDirect = true; + static string myDbpath = getParam( "dbpath" ); + mongo::dbpath = myDbpath.c_str(); + mongo::acquirePathLock(); } - cerr << "connected to: " << host << endl; - if ( _params.count( "db" ) ) _db = _params["db"].as(); diff --git a/tools/Tool.h b/tools/Tool.h index 8f7b2fa76c6..a579ad9afd6 100644 --- a/tools/Tool.h +++ b/tools/Tool.h @@ -11,6 +11,7 @@ #endif #include "client/dbclient.h" +#include "db/instance.h" using std::string; @@ -48,17 +49,21 @@ namespace mongo { } virtual int run() = 0; - + virtual void printExtraHelp( ostream & out ); protected: string _name; - mongo::DBClientConnection _conn; string _db; string _coll; + mongo::DBClientBase &conn() { return _useDirect ? (mongo::DBClientBase&)_direct : (mongo::DBClientBase&)_conn; }; + private: + mongo::DBClientConnection _conn; + mongo::DBDirectClient _direct; + bool _useDirect; boost::program_options::options_description * _options; boost::program_options::positional_options_description _positonalOptions; diff --git a/tools/dump.cpp b/tools/dump.cpp index 26604e15f0e..03bcb2c7b0e 100644 --- a/tools/dump.cpp +++ b/tools/dump.cpp @@ -40,7 +40,7 @@ public: int out = open( outputFile.string().c_str() , O_WRONLY | O_CREAT | O_TRUNC , 0666 ); assert( out ); - auto_ptr cursor = _conn.query( coll.c_str() , BSONObj() , 0 , 0 , 0 , Option_SlaveOk ); + auto_ptr cursor = conn().query( coll.c_str() , BSONObj() , 0 , 0 , 0 , Option_SlaveOk ); int num = 0; while ( cursor->more() ) { @@ -61,7 +61,7 @@ public: string sns = db + ".system.namespaces"; - auto_ptr cursor = _conn.query( sns.c_str() , BSONObj() , 0 , 0 , 0 , Option_SlaveOk ); + auto_ptr cursor = conn().query( sns.c_str() , BSONObj() , 0 , 0 , 0 , Option_SlaveOk ); while ( cursor->more() ) { BSONObj obj = cursor->next(); if ( obj.toString().find( ".$" ) != string::npos ) @@ -84,7 +84,7 @@ public: if ( db == "*" ){ cout << "all dbs" << endl; - BSONObj res = _conn.findOne( "admin.$cmd" , BSON( "listDatabases" << 1 ) ); + BSONObj res = conn().findOne( "admin.$cmd" , BSON( "listDatabases" << 1 ) ); BSONObj dbs = res.getField( "databases" ).embeddedObjectUserCheck(); set keys; dbs.getFieldNames( keys ); @@ -105,6 +105,7 @@ public: } return 0; } + }; int main( int argc , char ** argv ) { diff --git a/tools/export.cpp b/tools/export.cpp index 119f0be6d18..2245b7c7328 100644 --- a/tools/export.cpp +++ b/tools/export.cpp @@ -74,7 +74,7 @@ public: } - auto_ptr cursor = _conn.query( ns.c_str() , getParam( "query" , "" ) , 0 , 0 , fieldsToReturn , Option_SlaveOk ); + auto_ptr cursor = conn().query( ns.c_str() , getParam( "query" , "" ) , 0 , 0 , fieldsToReturn , Option_SlaveOk ); if ( csv ){ for ( vector::iterator i=fields.begin(); i != fields.end(); i++ ){ diff --git a/tools/files.cpp b/tools/files.cpp index 04b7dc0da4f..957c546105d 100644 --- a/tools/files.cpp +++ b/tools/files.cpp @@ -65,7 +65,7 @@ public: return -1; } - GridFS g( _conn , _db ); + GridFS g( conn() , _db ); string filename = getParam( "file" ); if ( cmd == "list" ){ diff --git a/tools/importJSON.cpp b/tools/importJSON.cpp index 7eb4f0a9fe9..28bd48b72ba 100644 --- a/tools/importJSON.cpp +++ b/tools/importJSON.cpp @@ -63,11 +63,11 @@ public: if ( hasParam( "drop" ) ){ cout << "dropping: " << ns << endl; - _conn.dropCollection( ns.c_str() ); + conn().dropCollection( ns.c_str() ); } if ( hasParam( "idbefore" ) ){ - _conn.ensureIndex( ns.c_str() , BSON( "_id" << 1 ) ); + conn().ensureIndex( ns.c_str() , BSON( "_id" << 1 ) ); } int num = 0; @@ -87,7 +87,7 @@ public: try { BSONObj o = fromjson( line ); - _conn.insert( ns.c_str() , o ); + conn().insert( ns.c_str() , o ); } catch ( MsgAssertionException& ma ){ cout << "exception:" << ma.toString() << endl; @@ -101,7 +101,7 @@ public: } if ( hasParam( "id" ) ){ - _conn.ensureIndex( ns.c_str() , BSON( "_id" << 1 ) ); + conn().ensureIndex( ns.c_str() , BSON( "_id" << 1 ) ); } return 0; diff --git a/tools/restore.cpp b/tools/restore.cpp index aa36b901ce1..56e360a1699 100644 --- a/tools/restore.cpp +++ b/tools/restore.cpp @@ -95,7 +95,7 @@ public: while ( read < mmf.length() ) { BSONObj o( data ); - _conn.insert( ns.c_str() , o ); + conn().insert( ns.c_str() , o ); read += o.objsize(); data += o.objsize(); From 11ae38a6ff5aa95115c0d16f5e6983edcc1606b5 Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 20 May 2009 12:15:26 -0400 Subject: [PATCH 08/16] use file allocator to allow writes in Tools MINOR --- tools/Tool.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/Tool.cpp b/tools/Tool.cpp index 8aec59632ce..257653d6342 100644 --- a/tools/Tool.cpp +++ b/tools/Tool.cpp @@ -6,6 +6,8 @@ #include +#include "util/file_allocator.h" + using namespace std; using namespace mongo; @@ -64,6 +66,9 @@ int mongo::Tool::main( int argc , char ** argv ){ static string myDbpath = getParam( "dbpath" ); mongo::dbpath = myDbpath.c_str(); mongo::acquirePathLock(); +#if !defined(_WIN32) + theFileAllocator().start(); +#endif } if ( _params.count( "db" ) ) From 3aa95556bd50e2f3fb671c0720dce31ca74e28fd Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 20 May 2009 12:15:41 -0400 Subject: [PATCH 09/16] better message when root missing MINOR --- tools/restore.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/restore.cpp b/tools/restore.cpp index 56e360a1699..269a95a7cfe 100644 --- a/tools/restore.cpp +++ b/tools/restore.cpp @@ -79,8 +79,10 @@ public: ns += "." + l; } - if ( boost::filesystem::file_size( root ) == 0 ) - return; + if ( boost::filesystem::file_size( root ) == 0 ) { + out() << "file " + root.native_file_string() + " empty, aborting" << endl; + return; + } out() << "\t going into namespace [" << ns << "]" << endl; From 919bdb31ce848612896dbfc8dfac529652329a3a Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 20 May 2009 12:16:25 -0400 Subject: [PATCH 10/16] add tool tests where db data files are accessed directly --- jstests/tool/tool1.js | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/jstests/tool/tool1.js b/jstests/tool/tool1.js index ba044f58fc0..7a9a7c6970f 100644 --- a/jstests/tool/tool1.js +++ b/jstests/tool/tool1.js @@ -1,36 +1,53 @@ // mongo tool tests, very basic to start with baseName = "jstests_tool_tool1"; -dbPath = "/data/db/" + baseName; -externalPath = "/data/db/" + baseName + "/external" -externalFile = externalPath + "/export.json" - +dbPath = "/data/db/" + baseName + "/"; +externalPath = "/data/db/" + baseName + "_external/" +externalFile = externalPath + "export.json" port = allocatePorts( 1 )[ 0 ]; +resetDbpath( externalPath ); m = startMongod( "--port", port, "--dbpath", dbPath, "--nohttpinterface" ); -resetDbpath( externalPath ); c = m.getDB( baseName ).getCollection( baseName ); c.save( { a: 1 } ); startMongoProgramNoConnect( "mongodump", "--host", "127.0.0.1:" + port, "--out", externalPath ); - sleep( 3000 ); - c.drop(); - startMongoProgramNoConnect( "mongorestore", "--host", "127.0.0.1:" + port, "--dir", externalPath ); - assert.soon( function() { return c.findOne() && 1 == c.findOne().a; } ); resetDbpath( externalPath ); startMongoProgramNoConnect( "mongoexport", "--host", "127.0.0.1:" + port, "-d", baseName, "-c", baseName, "--out", externalFile ); - sleep( 3000 ); - c.drop(); - startMongoProgramNoConnect( "mongoimportjson", "--host", "127.0.0.1:" + port, "-d", baseName, "-c", baseName, "--file", externalFile ); - assert.soon( function() { return c.findOne() && 1 == c.findOne().a; } ); + +stopMongod( port ); +resetDbpath( externalPath ); + +startMongoProgramNoConnect( "mongodump", "--dbpath", dbPath, "--out", externalPath ); +sleep( 3000 ); +resetDbpath( dbPath ); +startMongoProgramNoConnect( "mongorestore", "--dbpath", dbPath, "--dir", externalPath ); +sleep( 5000 ); +m = startMongoProgram( "mongod", "--port", port, "--dbpath", dbPath, "--nohttpinterface" ); +c = m.getDB( baseName ).getCollection( baseName ); +assert( c.findOne(), "object missing" ); +assert( 1 == c.findOne().a, "object wrong" ); + +stopMongod( port ); +resetDbpath( externalPath ); + +startMongoProgramNoConnect( "mongoexport", "--dbpath", dbPath, "-d", baseName, "-c", baseName, "--out", externalFile ); +sleep( 3000 ); +resetDbpath( dbPath ); +startMongoProgramNoConnect( "mongoimportjson", "--dbpath", dbPath, "-d", baseName, "-c", baseName, "--file", externalFile ); +sleep( 5000 ); +m = startMongoProgram( "mongod", "--port", port, "--dbpath", dbPath, "--nohttpinterface" ); +c = m.getDB( baseName ).getCollection( baseName ); +assert( c.findOne(), "object missing" ); +assert( 1 == c.findOne().a, "object wrong" ); From 333dbae2725567a9111dc4e8f9b962d21f75d08b Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 20 May 2009 12:19:33 -0400 Subject: [PATCH 11/16] adjust help formatting MINOR --- tools/export.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/export.cpp b/tools/export.cpp index f07ed84ea29..49e402d5149 100644 --- a/tools/export.cpp +++ b/tools/export.cpp @@ -36,8 +36,8 @@ class Export : public Tool { public: Export() : Tool( "export" ){ add_options() - ("query,q" , po::value() , " query filter" ) - ("fields,f" , po::value() , " comma seperated list of field names e.g. -f=name,age " ) + ("query,q" , po::value() , "query filter" ) + ("fields,f" , po::value() , "comma seperated list of field names e.g. -f=name,age " ) ("csv","export to csv instead of json") ("out,o", po::value(), "output file; if not specified, stdout is used") ; From 04a735fe263cd7afd02be3d8ff6047e857172bd9 Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 20 May 2009 13:09:54 -0400 Subject: [PATCH 12/16] debugging MINOR --- jstests/repl/repl6.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jstests/repl/repl6.js b/jstests/repl/repl6.js index 5c9c0db6de5..8447d698d92 100644 --- a/jstests/repl/repl6.js +++ b/jstests/repl/repl6.js @@ -4,10 +4,8 @@ var baseName = "jstests_repl6test"; soonCount = function( m, count ) { assert.soon( function() { - // print( "check count" ); - // print( "count: " + s.getDB( baseName ).z.find().count() ); return m.getDB( baseName ).a.find().count() == count; - } ); + }, "expected count: " + count + " from : " + m ); } doTest = function( signal ) { From a0a9411bc3483ce2e5f59242f84a8739abfd529c Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 20 May 2009 13:39:46 -0400 Subject: [PATCH 13/16] also expect port arg for mongobridge --- shell/utils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell/utils.cpp b/shell/utils.cpp index 50e6785f666..dd162a79e21 100644 --- a/shell/utils.cpp +++ b/shell/utils.cpp @@ -217,7 +217,7 @@ namespace mongo { } argv_[ args.nFields() ] = 0; - if ( program != "mongod" && program != "mongos" ) + if ( program != "mongod" && program != "mongos" && program != "mongobridge" ) port_ = 0; else assert( port_ > 0 ); From 1f6b10d7b32b898a570a82ceaa81cd8e28c85077 Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 20 May 2009 14:06:39 -0400 Subject: [PATCH 14/16] specify loopback interface when getting port --- shell/utils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell/utils.cpp b/shell/utils.cpp index dd162a79e21..9e8a7296b12 100644 --- a/shell/utils.cpp +++ b/shell/utils.cpp @@ -124,7 +124,7 @@ namespace mongo { memset(address.sin_zero, 0, sizeof(address.sin_zero)); address.sin_family = AF_INET; address.sin_port = 0; - address.sin_addr.s_addr = 0; + address.sin_addr.s_addr = inet_addr( "127.0.0.1" ); assert( 0 == ::bind( s, (sockaddr*)&address, sizeof( address ) ) ); sockaddr_in newAddress; From 3b2ae5b7fb7bc9b869a5d799f6966250db0d40d9 Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 20 May 2009 14:07:10 -0400 Subject: [PATCH 15/16] close mongobridge's listener socket when program killed --- tools/bridge.cpp | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/tools/bridge.cpp b/tools/bridge.cpp index 3ec9450abb9..63a576accec 100644 --- a/tools/bridge.cpp +++ b/tools/bridge.cpp @@ -79,8 +79,31 @@ void check( bool b ) { helpExit(); } -int main( int argc, char **argv ) { +auto_ptr< MyListener > listener; +#if !defined(_WIN32) +#include +void cleanup( int sig ) { + close( listener->socket() ); + ::exit( 0 ); +} + +void setupSignals() { + signal( SIGINT , cleanup ); + signal( SIGTERM , cleanup ); + signal( SIGPIPE , cleanup ); + signal( SIGABRT , cleanup ); + signal( SIGSEGV , cleanup ); + signal( SIGBUS , cleanup ); + signal( SIGFPE , cleanup ); +} +#else +inline void setupSignals() {} +#endif + +int main( int argc, char **argv ) { + setupSignals(); + check( argc == 5 ); for( int i = 1; i < 5; ++i ) { @@ -95,9 +118,9 @@ int main( int argc, char **argv ) { } check( port != 0 && !destUri.empty() ); - MyListener l( port ); - l.init(); - l.listen(); + listener.reset( new MyListener( port ) ); + listener->init(); + listener->listen(); return 0; } From c5ffd65fb28fb6b201bb6027cb0ef0b52fefcf8c Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 20 May 2009 14:16:14 -0400 Subject: [PATCH 16/16] mongobridge close client ports on kill --- tools/bridge.cpp | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/tools/bridge.cpp b/tools/bridge.cpp index 63a576accec..f997ffd6c80 100644 --- a/tools/bridge.cpp +++ b/tools/bridge.cpp @@ -58,33 +58,26 @@ private: MessagingPort &mp_; }; +set ports; + class MyListener : public Listener { public: MyListener( int port ) : Listener( "", port ) {} virtual void accepted(MessagingPort *mp) { + ports.insert( mp ); Forwarder f( *mp ); boost::thread t( f ); } }; -void helpExit() { - cout << "usage mongobridge --port --dest " << endl; - cout << " port: port to listen for mongo messages" << endl; - cout << " destUri: uri of remote mongod instance" << endl; - ::exit( -1 ); -} - -void check( bool b ) { - if ( !b ) - helpExit(); -} - auto_ptr< MyListener > listener; #if !defined(_WIN32) #include void cleanup( int sig ) { close( listener->socket() ); + for ( set::iterator i = ports.begin(); i != ports.end(); i++ ) + (*i)->shutdown(); ::exit( 0 ); } @@ -101,6 +94,18 @@ void setupSignals() { inline void setupSignals() {} #endif +void helpExit() { + cout << "usage mongobridge --port --dest " << endl; + cout << " port: port to listen for mongo messages" << endl; + cout << " destUri: uri of remote mongod instance" << endl; + ::exit( -1 ); +} + +void check( bool b ) { + if ( !b ) + helpExit(); +} + int main( int argc, char **argv ) { setupSignals();