From 15aebb07066889ff7fbf459f36d4830be0a925b6 Mon Sep 17 00:00:00 2001 From: Eliot Horowitz Date: Tue, 15 Jun 2010 23:05:45 -0400 Subject: [PATCH] mongod chunk filtering seems to be working SERVER-943 --- db/query.cpp | 2 +- jstests/sharding/key_many.js | 2 +- s/commands_public.cpp | 1 - s/d_logic.h | 2 +- s/d_state.cpp | 67 ++++++++++++++++++++---------------- s/strategy_shard.cpp | 4 +-- 6 files changed, 42 insertions(+), 36 deletions(-) diff --git a/db/query.cpp b/db/query.cpp index 0fb08624d34..99048a3ffeb 100644 --- a/db/query.cpp +++ b/db/query.cpp @@ -637,7 +637,7 @@ namespace mongo { _nscannedObjects++; DiskLoc cl = _c->currLoc(); if ( _chunkMatcher && ! _chunkMatcher->belongsToMe( _c->currKey(), _c->currLoc() ) ){ - cout << "TEMP skipping un-owned chunk: " << _c->current() << endl; + // cout << "TEMP skipping un-owned chunk: " << _c->current() << endl; } else if( _c->getsetdup(cl) ) { // dup diff --git a/jstests/sharding/key_many.js b/jstests/sharding/key_many.js index 1c17c41ca54..4931d981cde 100644 --- a/jstests/sharding/key_many.js +++ b/jstests/sharding/key_many.js @@ -7,7 +7,7 @@ types = [ { name : "double" , values : [ 1.2 , 3.5 , 4.5 , 4.6 , 6.7 , 9.9 ] , keyfield : "a" } , { name : "date" , values : [ new Date( 1000000 ) , new Date( 2000000 ) , new Date( 3000000 ) , new Date( 4000000 ) , new Date( 5000000 ) , new Date( 6000000 ) ] , keyfield : "a" } , { name : "string_id" , values : [ "allan" , "bob" , "eliot" , "joe" , "mark" , "sara" ] , keyfield : "_id" }, - { name : "embedded" , values : [ "allan" , "bob" , "eliot" , "joe" , "mark" , "sara" ] , keyfield : "a.b" } , + { name : "embedded 1" , values : [ "allan" , "bob" , "eliot" , "joe" , "mark" , "sara" ] , keyfield : "a.b" } , { name : "embedded 2" , values : [ "allan" , "bob" , "eliot" , "joe" , "mark" , "sara" ] , keyfield : "a.b.c" } , { name : "object" , values : [ {a:1, b:1.2}, {a:1, b:3.5}, {a:1, b:4.5}, {a:2, b:1.2}, {a:2, b:3.5}, {a:2, b:4.5} ] , keyfield : "o" } , { name : "oid_id" , values : [ ObjectId() , ObjectId() , ObjectId() , ObjectId() , ObjectId() , ObjectId() ] , keyfield : "_id" } , diff --git a/s/commands_public.cpp b/s/commands_public.cpp index b4dbe2f476f..366e05ab10e 100644 --- a/s/commands_public.cpp +++ b/s/commands_public.cpp @@ -315,7 +315,6 @@ namespace mongo { shared_ptr c = *i; BSONObj myCommand = fixCmdObj( cmdObj , c ); - cout << "myCommand: " << myCommand << endl; ShardConnection conn( c->getShard() , fullns ); BSONObj res; bool ok = conn->runCommand( conf->getName() , myCommand , res ); diff --git a/s/d_logic.h b/s/d_logic.h index 3726e9b9972..628c06cf1b2 100644 --- a/s/d_logic.h +++ b/s/d_logic.h @@ -75,7 +75,7 @@ namespace mongo { void appendInfo( BSONObjBuilder& b ); - ChunkMatcherPtr getChunkMatcher( const string& ns , bool load=false , ConfigVersion version=0 ); + ChunkMatcherPtr getChunkMatcher( const string& ns ); private: diff --git a/s/d_state.cpp b/s/d_state.cpp index 77bd1b18e63..ac000760e30 100644 --- a/s/d_state.cpp +++ b/s/d_state.cpp @@ -138,38 +138,24 @@ namespace mongo { } - ChunkMatcherPtr ShardingState::getChunkMatcher( const string& ns , bool load , ConfigVersion version ){ - return ChunkMatcherPtr();// ERH ERH ERH - + ChunkMatcherPtr ShardingState::getChunkMatcher( const string& ns ){ if ( ! _enabled ) return ChunkMatcherPtr(); if ( ! ShardedConnectionInfo::get( false ) ) return ChunkMatcherPtr(); - + + ConfigVersion version; { scoped_lock lk( _mutex ); - if ( ! version ) - version = _versions[ns]; + version = _versions[ns]; + if ( ! version ) return ChunkMatcherPtr(); ChunkMatcherPtr p = _chunks[ns]; - if ( p ){ - if ( ! load ) - return p; - - if ( p->_version >= version ) - return p; - } - else { - if ( ! load ){ - stringstream ss; - ss << "getChunkMatcher called with load false for ns: " << ns; - msgasserted( 13300 , ss.str() ); - } - } - + if ( p && p->_version >= version ) + return p; } BSONObj q; @@ -179,12 +165,25 @@ namespace mongo { b.append( "shard" , BSON( "$in" << BSON_ARRAY( _shardHost << _shardName ) ) ); q = b.obj(); } + + auto_ptr scoped; + auto_ptr direct; - ScopedDbConnection conn( _configServer ); + DBClientBase * conn; + + if ( _configServer == _shardHost ){ + direct.reset( new DBDirectClient() ); + conn = direct.get(); + } + else { + scoped.reset( new ScopedDbConnection( _configServer ) ); + conn = scoped->get(); + } auto_ptr cursor = conn->query( "config.chunks" , Query(q).sort( "min" ) ); if ( ! cursor->more() ){ - conn.done(); + if ( scoped.get() ) + scoped->done(); return ChunkMatcherPtr(); } @@ -212,7 +211,8 @@ namespace mongo { assert( ! min.isEmpty() ); p->gotRange( min.getOwned() , max.getOwned() ); - conn.done(); + if ( scoped.get() ) + scoped->done(); { scoped_lock lk( _mutex ); @@ -420,7 +420,7 @@ namespace mongo { { dbtemprelease unlock; - shardingState.getChunkMatcher( ns , true , version ); + shardingState.getChunkMatcher( ns ); } result.appendTimestamp( "oldVersion" , oldVersion ); @@ -534,17 +534,24 @@ namespace mongo { } bool ChunkMatcher::belongsToMe( const BSONObj& key , const DiskLoc& loc ) const { - return true; // ERH ERH ERH if ( _map.size() == 0 ) return false; - BSONObj x = loc.obj().getFieldDotted( _field.c_str() ).wrap(); // TODO: this is slow + BSONObj x = loc.obj().getFieldDotted( _field.c_str() ).wrap( _field.c_str() ); // TODO: this is slow MyMap::const_iterator a = _map.upper_bound( x ); - if ( a == _map.end() ) - a--; + a--; - return x.woCompare( a->second.first ) >= 0 && x.woCompare( a->second.second ) < 0; + bool good = x.woCompare( a->second.first ) >= 0 && x.woCompare( a->second.second ) < 0; +#if 0 + if ( ! good ){ + cout << "bad: " << x << "\t" << a->second.first << "\t" << x.woCompare( a->second.first ) << "\t" << x.woCompare( a->second.second ) << endl; + for ( MyMap::const_iterator i=_map.begin(); i!=_map.end(); ++i ){ + cout << "\t" << i->first << "\t" << i->second.first << "\t" << i->second.second << endl; + } + } +#endif + return good; } } diff --git a/s/strategy_shard.cpp b/s/strategy_shard.cpp index 168ccd756f7..f8b454ce53f 100644 --- a/s/strategy_shard.cpp +++ b/s/strategy_shard.cpp @@ -50,8 +50,8 @@ namespace mongo { set servers; for ( vector >::iterator i = shards.begin(); i != shards.end(); i++ ){ shared_ptr c = *i; - //servers.insert( ServerAndQuery( c->getShard().getConnString() , BSONObj() ) ); // ERH ERH ERH - servers.insert( ServerAndQuery( c->getShard().getConnString() , c->getFilter() ) ); + servers.insert( ServerAndQuery( c->getShard().getConnString() , BSONObj() ) ); + //servers.insert( ServerAndQuery( c->getShard().getConnString() , c->getFilter() ) ); // this is what does mongod size filtering, TODO: clean up apis } if ( logLevel > 4 ){