From e0727dc6580dc8d9e53a7047fe9ce3cd6a62cca9 Mon Sep 17 00:00:00 2001 From: Dwight Date: Wed, 3 Dec 2008 18:27:32 -0500 Subject: [PATCH] clone/copy db now works on replicating databases --- db/cloner.cpp | 32 +++++++++++++++++++++----------- db/dbcommands.cpp | 5 +---- db/jsobj.h | 9 +++++++++ db/pdfile.cpp | 17 +++++++++++++---- db/pdfile.h | 3 ++- db/repl.cpp | 7 ++----- db/repl.h | 2 +- 7 files changed, 49 insertions(+), 26 deletions(-) diff --git a/db/cloner.cpp b/db/cloner.cpp index 5e1952abcc4..1b4da981a79 100644 --- a/db/cloner.cpp +++ b/db/cloner.cpp @@ -24,21 +24,22 @@ #include "query.h" #include "commands.h" #include "db.h" +#include "repl.h" extern int port; -bool userCreateNS(const char *ns, BSONObj& j, string& err); class Cloner: boost::noncopyable { DBClientConnection conn; void copy(const char *from_ns, const char *to_ns, bool isindex = false); public: Cloner() { } - bool go(const char *masterHost, string& errmsg, const string& fromdb); + bool go(const char *masterHost, string& errmsg, const string& fromdb, bool logForRepl); }; /* for index info object: { "name" : "name_1" , "ns" : "foo.index3" , "key" : { "name" : 1.0 } } - we need to fix up the value in the "ns" parameter. + we need to fix up the value in the "ns" parameter so that the name prefix is correct on a + copy to a new name. */ BSONObj fixindex(BSONObj o) { BSONObjBuilder b; @@ -69,6 +70,9 @@ BSONObj fixindex(BSONObj o) { return res; } +/* copy the specified collection + isindex - if true, this is system.indexes collection. +*/ void Cloner::copy(const char *from_collection, const char *to_collection, bool isindex) { auto_ptr c; { @@ -91,13 +95,17 @@ void Cloner::copy(const char *from_collection, const char *to_collection, bool i } BSONObj js = tmp; - if( isindex ) + if( isindex ) { + assert( strstr(from_collection, "system.indexes") ); js = fixindex(tmp); + } + theDataFileMgr.insert(to_collection, (void*) js.objdata(), js.objsize()); + logOp("i", to_collection, js); } } -bool Cloner::go(const char *masterHost, string& errmsg, const string& fromdb) { +bool Cloner::go(const char *masterHost, string& errmsg, const string& fromdb, bool logForRepl) { string todb = database->name; stringstream a,b; a << "localhost:" << port; @@ -150,7 +158,7 @@ bool Cloner::go(const char *masterHost, string& errmsg, const string& fromdb) { //if( !options.isEmpty() ) { string err; - userCreateNS(to_name.c_str(), options, err); + userCreateNS(to_name.c_str(), options, err, logForRepl); } copy(from_name, to_name.c_str()); } @@ -163,10 +171,10 @@ bool Cloner::go(const char *masterHost, string& errmsg, const string& fromdb) { return true; } -bool cloneFrom(const char *masterHost, string& errmsg, const string& fromdb) +bool cloneFrom(const char *masterHost, string& errmsg, const string& fromdb, bool logForReplication) { Cloner c; - return c.go(masterHost, errmsg, fromdb); + return c.go(masterHost, errmsg, fromdb, logForReplication); } /* Usage: @@ -180,7 +188,10 @@ public: string from = cmdObj.getStringField("clone"); if( from.empty() ) return false; - return cloneFrom(from.c_str(), errmsg, database->name); + /* replication note: we must logOp() not the command, but the cloned data -- if the slave + were to clone it would get a different point-in-time and not match. + */ + return cloneFrom(from.c_str(), errmsg, database->name, true); } } cmdclone; @@ -207,9 +218,8 @@ public: return false; } setClient(todb.c_str()); - bool res = cloneFrom(fromhost.c_str(), errmsg, fromdb); + bool res = cloneFrom(fromhost.c_str(), errmsg, fromdb, true); database = 0; return res; } } cmdcopydb; - diff --git a/db/dbcommands.cpp b/db/dbcommands.cpp index 2f7ebe0929c..d9d76b3689c 100644 --- a/db/dbcommands.cpp +++ b/db/dbcommands.cpp @@ -31,7 +31,6 @@ extern int queryTraceLevel; extern int otherTraceLevel; extern int opLogging; -bool userCreateNS(const char *ns, BSONObj& j, string& err); void flushOpLog(); int runCount(const char *ns, BSONObj& cmd, string& err); @@ -319,9 +318,7 @@ bool _runCommands(const char *ns, BSONObj& jsobj, stringstream& ss, BufBuilder & valid = true; string ns = us + '.' + e.valuestr(); string err; - ok = userCreateNS(ns.c_str(), jsobj, err); - if( ok ) - logOp("c", ns.c_str(), jsobj); + ok = userCreateNS(ns.c_str(), jsobj, err, true); if( !ok && !err.empty() ) anObjBuilder.append("errmsg", err.c_str()); } diff --git a/db/jsobj.h b/db/jsobj.h index eb0b705f27c..448bac93325 100644 --- a/db/jsobj.h +++ b/db/jsobj.h @@ -300,6 +300,9 @@ public: int objsize() const { return details ? details->_objsize : 0; } // includes the embedded size field bool isEmpty() const { return objsize() <= 5; } + /* sigh...details == 0 is such a pain we have to eliminate that possibility */ + void validateEmpty(); + void dump() { cout << hex; const char *p = objdata(); @@ -620,3 +623,9 @@ inline void BSONObjBuilder::appendElements(BSONObj x) { } extern BSONObj emptyObj; + +inline void BSONObj::validateEmpty() { + if( details == 0 ) + *this = emptyObj; +} + diff --git a/db/pdfile.cpp b/db/pdfile.cpp index 23d03351ba0..3e9cead7661 100644 --- a/db/pdfile.cpp +++ b/db/pdfile.cpp @@ -33,6 +33,7 @@ _ disallow system* manipulations from the database. #include #include #include "query.h" +#include "repl.h" const char *dbpath = "/data/db/"; @@ -92,9 +93,7 @@ int initialExtentSize(int len) { return z; } -// { ..., capped: true, size: ..., max: ... } -// returns true if successful -bool userCreateNS(const char *ns, BSONObj& j, string& err) { +bool _userCreateNS(const char *ns, BSONObj& j, string& err) { if( nsdetails(ns) ) { err = "collection already exists"; return false; @@ -134,6 +133,16 @@ bool userCreateNS(const char *ns, BSONObj& j, string& err) { return true; } +// { ..., capped: true, size: ..., max: ... } +// returns true if successful +bool userCreateNS(const char *ns, BSONObj j, string& err, bool logForReplication) { + j.validateEmpty(); + bool ok = _userCreateNS(ns, j, err); + if( logForReplication && ok ) + logOp("c", ns, j); + return ok; +} + /*---------------------------------------------------------------------*/ void PhysicalDataFile::open(int fn, const char *filename) { @@ -752,7 +761,7 @@ DiskLoc DataFileMgr::insert(const char *ns, const void *buf, int len, bool god) if( tableToIndex == 0 ) { // try to create it string err; - if( !userCreateNS(tabletoidxns.c_str(), emptyObj, err) ) { + if( !userCreateNS(tabletoidxns.c_str(), emptyObj, err, false) ) { problem() << "ERROR: failed to create collection while adding its index. " << tabletoidxns << endl; return DiskLoc(); } diff --git a/db/pdfile.h b/db/pdfile.h index 43650a61b67..085653894a0 100644 --- a/db/pdfile.h +++ b/db/pdfile.h @@ -40,7 +40,8 @@ class Record; class Cursor; void dropDatabase(const char *ns); -void dropNS(string& dropNs); +void dropNS(string& dropNs);; +bool userCreateNS(const char *ns, BSONObj j, string& err, bool logForReplication); /*---------------------------------------------------------------------*/ diff --git a/db/repl.cpp b/db/repl.cpp index 7259f662096..0318768d7dc 100644 --- a/db/repl.cpp +++ b/db/repl.cpp @@ -37,7 +37,6 @@ extern boost::mutex dbMutex; auto_ptr findTableScan(const char *ns, BSONObj& order, bool *isSorted=0); -bool userCreateNS(const char *ns, BSONObj& j, string& err); int _updateObjects(const char *ns, BSONObj updateobj, BSONObj pattern, bool upsert, stringstream& ss, bool logOp=false); bool _runCommands(const char *ns, BSONObj& jsobj, stringstream& ss, BufBuilder &b, BSONObjBuilder& anObjBuilder); void ensureHaveIdIndex(const char *ns); @@ -393,10 +392,8 @@ bool ReplSource::resync(string db) { { log() << "resync: cloning database " << db << endl; - //Cloner c; string errmsg; - bool ok = cloneFrom(hostName.c_str(), errmsg, database->name); - //bool ok = c.go(hostName.c_str(), errmsg); + bool ok = cloneFrom(hostName.c_str(), errmsg, database->name, false); if( !ok ) { problem() << "resync of " << db << " from " << hostName << " failed " << errmsg << endl; throw SyncException(); @@ -929,7 +926,7 @@ void startReplication() { setClientTempNs("local.oplog.$main"); string err; BSONObj o = b.done(); - userCreateNS("local.oplog.$main", o, err); + userCreateNS("local.oplog.$main", o, err, false); database = 0; } diff --git a/db/repl.h b/db/repl.h index 5cddf41e985..be0722fe089 100644 --- a/db/repl.h +++ b/db/repl.h @@ -35,7 +35,7 @@ class DBClientCursor; extern bool slave; extern bool master; -bool cloneFrom(const char *masterHost, string& errmsg, const string& fromdb); +bool cloneFrom(const char *masterHost, string& errmsg, const string& fromdb, bool logForReplication); #pragma pack(push,4) class OpTime {