mirrors/mongodb
mirrors/mongodb
0
0
Fork 0
mirror of https://github.com/mongodb/mongo.git synced 2024-12-01 09:32:32 +01:00
Code Issues Releases Activity

use better _id for chunks

Browse Source
This commit is contained in:
Eliot Horowitz 2010-04-22 10:29:05 -04:00
parent 835c65adc2
commit a3242b6791
5 changed files with 215 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
SConstruct
View File

@ -407,7 +407,7 @@ else:
nojni = True
coreShardFiles = []
shardServerFiles = coreShardFiles + Glob( "s/strategy*.cpp" ) + [ "s/commands_admin.cpp" , "s/commands_public.cpp" , "s/request.cpp" , "s/cursors.cpp" , "s/server.cpp" , "s/chunk.cpp" , "s/shardkey.cpp" , "s/config.cpp" , "s/s_only.cpp" , "s/stats.cpp" , "db/cmdline.cpp" ]
shardServerFiles = coreShardFiles + Glob( "s/strategy*.cpp" ) + [ "s/commands_admin.cpp" , "s/commands_public.cpp" , "s/request.cpp" , "s/cursors.cpp" , "s/server.cpp" , "s/chunk.cpp" , "s/shardkey.cpp" , "s/config.cpp" , "s/config_migrate.cpp" , "s/s_only.cpp" , "s/stats.cpp" , "db/cmdline.cpp" ]
serverOnlyFiles += coreShardFiles + [ "s/d_logic.cpp" ]
serverOnlyFiles += [ "db/module.cpp" ] + Glob( "db/modules/*.cpp" )

16
s/chunk.cpp
View File

@ -356,6 +356,9 @@ namespace mongo {
}
void Chunk::serialize(BSONObjBuilder& to){
to.append( "_id" , genID( _ns , _min ) );
if ( _lastmod )
to.appendTimestamp( "lastmod" , _lastmod );
else
@ -366,6 +369,19 @@ namespace mongo {
to << "max" << _max;
to << "shard" << _shard;
}
string Chunk::genID( const string& ns , const BSONObj& o ){
StringBuilder buf( ns.size() + o.objsize() + 16 );
buf << ns << "-";
BSONObjIterator i(o);
while ( i.more() ){
BSONElement e = i.next();
buf << e.fieldName() << "_" << e.toString( false );
}
return buf.str();
}
void Chunk::unserialize(const BSONObj& from){
_ns = from.getStringField( "ns" );

2
s/chunk.h
View File

@ -123,6 +123,8 @@ namespace mongo {
static int MaxChunkSize;
static string genID( const string& ns , const BSONObj& min );
private:
// main shard info

122
s/config.cpp
View File

@ -453,128 +453,6 @@ namespace mongo {
return version;
}
int ConfigServer::checkConfigVersion( bool upgrade ){
int cur = dbConfigVersion();
if ( cur == VERSION )
return 0;
if ( cur == 0 ){
ShardConnection conn( _primary );
conn->insert( "config.version" , BSON( "_id" << 1 << "version" << VERSION ) );
pool.flush();
assert( VERSION == dbConfigVersion( conn.conn() ) );
conn.done();
return 0;
}
if ( cur == 2 ){
// need to upgrade
assert( VERSION == 3 );
if ( ! upgrade ){
log() << "newer version of mongo meta data\n"
<< "need to --upgrade after shutting all mongos down"
<< endl;
return -9;
}
ShardConnection conn( _primary );
// do a backup
string backupName;
{
stringstream ss;
ss << "config-backup-" << terseCurrentTime();
backupName = ss.str();
}
log() << "backing up config to: " << backupName << endl;
conn->copyDatabase( "config" , backupName );
map<string,string> hostToShard;
set<string> shards;
// shards
{
unsigned n = 0;
auto_ptr<DBClientCursor> c = conn->query( ShardNS::shard , BSONObj() );
while ( c->more() ){
BSONObj o = c->next();
string host = o["host"].String();
string name = "";
BSONElement id = o["_id"];
if ( id.type() == String ){
name = id.String();
}
else {
stringstream ss;
ss << "shard" << hostToShard.size();
name = ss.str();
}
hostToShard[host] = name;
shards.insert( name );
n++;
}
assert( n == hostToShard.size() );
assert( n == shards.size() );
conn->remove( ShardNS::shard , BSONObj() );
for ( map<string,string>::iterator i=hostToShard.begin(); i != hostToShard.end(); i++ ){
conn->insert( ShardNS::shard , BSON( "_id" << i->second << "host" << i->first ) );
}
}
// databases
{
auto_ptr<DBClientCursor> c = conn->query( ShardNS::database , BSONObj() );
map<string,BSONObj> newDBs;
unsigned n = 0;
while ( c->more() ){
BSONObj old = c->next();
BSONObjBuilder b(old.objsize());
b.appendAs( old["name"] , "_id" );
BSONObjIterator i(old);
while ( i.more() ){
BSONElement e = i.next();
if ( strcmp( "_id" , e.fieldName() ) == 0 ||
strcmp( "name" , e.fieldName() ) == 0 ){
continue;
}
b.append( e );
}
BSONObj x = b.obj();
cout << old << "\n\t" << x << endl;
newDBs[old["name"].String()] = x;
n++;
}
assert( n == newDBs.size() );
conn->remove( ShardNS::database , BSONObj() );
for ( map<string,BSONObj>::iterator i=newDBs.begin(); i!=newDBs.end(); i++ ){
conn->insert( ShardNS::database , i->second );
}
}
conn->update( "config.version" , BSONObj() , BSON( "_id" << 1 << "version" << VERSION ) );
conn.done();
pool.flush();
return 1;
}
log() << "don't know how to upgrade " << cur << " to " << VERSION << endl;
return -8;
}
void ConfigServer::reloadSettings(){
set<string> got;

196
s/config_migrate.cpp Normal file
View File

@ -0,0 +1,196 @@
// config_migrate.cpp
/**
* Copyright (C) 2008 10gen Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "stdafx.h"
#include "../util/message.h"
#include "../util/unittest.h"
#include "../client/connpool.h"
#include "../client/model.h"
#include "../db/pdfile.h"
#include "../db/cmdline.h"
#include "server.h"
#include "config.h"
#include "chunk.h"
namespace mongo {
int ConfigServer::checkConfigVersion( bool upgrade ){
int cur = dbConfigVersion();
if ( cur == VERSION )
return 0;
if ( cur == 0 ){
ShardConnection conn( _primary );
conn->insert( "config.version" , BSON( "_id" << 1 << "version" << VERSION ) );
pool.flush();
assert( VERSION == dbConfigVersion( conn.conn() ) );
conn.done();
return 0;
}
if ( cur == 2 ){
// need to upgrade
assert( VERSION == 3 );
if ( ! upgrade ){
log() << "newer version of mongo meta data\n"
<< "need to --upgrade after shutting all mongos down"
<< endl;
return -9;
}
ShardConnection conn( _primary );
// do a backup
string backupName;
{
stringstream ss;
ss << "config-backup-" << terseCurrentTime();
backupName = ss.str();
}
log() << "backing up config to: " << backupName << endl;
conn->copyDatabase( "config" , backupName );
map<string,string> hostToShard;
set<string> shards;
// shards
{
unsigned n = 0;
auto_ptr<DBClientCursor> c = conn->query( ShardNS::shard , BSONObj() );
while ( c->more() ){
BSONObj o = c->next();
string host = o["host"].String();
string name = "";
BSONElement id = o["_id"];
if ( id.type() == String ){
name = id.String();
}
else {
stringstream ss;
ss << "shard" << hostToShard.size();
name = ss.str();
}
hostToShard[host] = name;
shards.insert( name );
n++;
}
assert( n == hostToShard.size() );
assert( n == shards.size() );
conn->remove( ShardNS::shard , BSONObj() );
for ( map<string,string>::iterator i=hostToShard.begin(); i != hostToShard.end(); i++ ){
conn->insert( ShardNS::shard , BSON( "_id" << i->second << "host" << i->first ) );
}
}
// databases
{
auto_ptr<DBClientCursor> c = conn->query( ShardNS::database , BSONObj() );
map<string,BSONObj> newDBs;
unsigned n = 0;
while ( c->more() ){
BSONObj old = c->next();
n++;
if ( old["name"].eoo() ){
// already done
newDBs[old["_id"].String()] = old;
continue;
}
BSONObjBuilder b(old.objsize());
b.appendAs( old["name"] , "_id" );
BSONObjIterator i(old);
while ( i.more() ){
BSONElement e = i.next();
if ( strcmp( "_id" , e.fieldName() ) == 0 ||
strcmp( "name" , e.fieldName() ) == 0 ){
continue;
}
b.append( e );
}
BSONObj x = b.obj();
cout << old << "\n\t" << x << endl;
newDBs[old["name"].String()] = x;
}
assert( n == newDBs.size() );
conn->remove( ShardNS::database , BSONObj() );
for ( map<string,BSONObj>::iterator i=newDBs.begin(); i!=newDBs.end(); i++ ){
conn->insert( ShardNS::database , i->second );
}
}
// chunks
{
unsigned num = 0;
map<string,BSONObj> chunks;
auto_ptr<DBClientCursor> c = conn->query( ShardNS::chunk , BSONObj() );
while ( c->more() ){
BSONObj x = c->next();
BSONObjBuilder b;
string id = Chunk::genID( x["ns"].String() , x["min"].Obj() );
b.append( "_id" , id );
BSONObjIterator i(x);
while ( i.more() ){
BSONElement e = i.next();
if ( strcmp( e.fieldName() , "_id" ) == 0 )
continue;
b.append( e );
}
BSONObj n = b.obj();
cout << x << "\n\t" << n << endl;
chunks[id] = n;
num++;
}
assert( num == chunks.size() );
conn->remove( ShardNS::chunk , BSONObj() );
for ( map<string,BSONObj>::iterator i=chunks.begin(); i!=chunks.end(); i++ ){
conn->insert( ShardNS::chunk , i->second );
}
}
conn->update( "config.version" , BSONObj() , BSON( "_id" << 1 << "version" << VERSION ) );
conn.done();
pool.flush();
return 1;
}
log() << "don't know how to upgrade " << cur << " to " << VERSION << endl;
return -8;
}
}