mirrors/mongodb
mirrors/mongodb
0
0
Fork 0
mirror of https://github.com/mongodb/mongo.git synced 2024-12-01 09:32:32 +01:00
Code Issues Releases Activity

btreebuilder work / temp commit

Browse Source
This commit is contained in:
Dwight 2009-09-24 12:11:55 -04:00
parent 4d9812894b
commit 6cf3d75d08
4 changed files with 186 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

129
db/btree.cpp
View File

@ -202,6 +202,20 @@ namespace mongo {
setNotPacked();
}
/* pull rightmost key from the bucket
*/
void BucketBasics::popBack(DiskLoc& recLoc, BSONObj& key, DiskLoc& rchild) {
massert( "n==0 in btree popBack()", n > 0 );
assert( k(n-1).isUsed() ); // no unused skipping in this function at this point - btreebuilder doesn't require that
KeyNode kn = keyNode(n-1);
recLoc = kn.recordLoc;
key = kn.key;
DiskLoc& rc = childForPos(n);
rchild = rc;
rc.Null();
n--;
}
/* add a key. must be > all existing. be careful to set next ptr right. */
void BucketBasics::pushBack(const DiskLoc& recordLoc, BSONObj& key, const BSONObj &order, DiskLoc prevChild) {
int bytesNeeded = key.objsize() + sizeof(_KeyNode);
@ -215,6 +229,10 @@ namespace mongo {
char *p = dataAt(kn.keyDataOfs());
memcpy(p, key.objdata(), key.objsize());
}
/*void BucketBasics::pushBack(const DiskLoc& recordLoc, BSONObj& key, const BSONObj &order, DiskLoc prevChild, DiskLoc nextChild) {
pushBack(recordLoc, key, order, prevChild);
childForPos(n) = nextChild;
}*/
/* insert a key in a bucket with no complexity -- no splits required */
bool BucketBasics::basicInsert(const DiskLoc& thisLoc, int keypos, const DiskLoc& recordLoc, const BSONObj& key, const BSONObj &order) {
@ -646,16 +664,16 @@ found:
}
/* start a new index off, empty */
DiskLoc BtreeBucket::addHead(IndexDetails& id) {
DiskLoc BtreeBucket::addBucket(IndexDetails& id) {
BtreeBucket *p = allocTemp();
DiskLoc loc = btreeStore->insert(id.indexNamespace().c_str(), p, p->Size(), true);
free(p);
return loc;
}
void BtreeBucket::renameIndexNamespace(const char *oldNs, const char *newNs) {
btreeStore->rename( oldNs, newNs );
}
void BtreeBucket::renameIndexNamespace(const char *oldNs, const char *newNs) {
btreeStore->rename( oldNs, newNs );
}
DiskLoc BtreeBucket::getHead(const DiskLoc& thisLoc) {
DiskLoc p = thisLoc;
@ -678,7 +696,7 @@ found:
if ( !nextDown.isNull() ) {
while ( 1 ) {
keyOfs = direction>0 ? 0 : nextDown.btree()->n - 1;
DiskLoc loc= nextDown.btree()->childForPos(keyOfs + adj);
DiskLoc loc = nextDown.btree()->childForPos(keyOfs + adj);
if ( loc.isNull() )
break;
nextDown = loc;
@ -776,7 +794,7 @@ found:
}
DEBUGGING out() << "TEMP: key: " << key.toString() << endl;
DiskLoc& child = getChild(pos);
DiskLoc& child = childForPos(pos);
if ( insert_debug )
out() << " getChild(" << pos << "): " << child.toString() << endl;
if ( child.isNull() || !rChild.isNull() /* means an 'internal' insert */ ) {
@ -858,7 +876,7 @@ namespace mongo {
b->dumpTree(id.head, order);
/* b->bt_insert(id.head, B, key, order, false, id);
/* b->bt_insert(id.head, B, key, order, false, id);
b->k(1).setUnused();
b->dumpTree(id.head, order);
@ -875,4 +893,101 @@ namespace mongo {
b->dumpTree(id.head, order);
}
/* --- BtreeBuilder --- */
BtreeBuilder::BtreeBuilder(bool _dupsAllowed, IndexDetails& _idx) :
dupsAllowed(_dupsAllowed), idx(_idx), n(0)
{
first = cur = BtreeBucket::addBucket(idx);
b = cur.btreemod();
order = idx.keyPattern();
committed = false;
}
void BtreeBuilder::newBucket() {
DiskLoc L = BtreeBucket::addBucket(idx);
b->tempNext() = L;
cur = L;
b = cur.btreemod();
}
void BtreeBuilder::addKey(BSONObj& key, DiskLoc loc) {
if( n > 0 ) {
int cmp = keyLast.woCompare(key);
massert( "bad key order in BtreeBuilder - server internal error", cmp >= 0 );
if( !dupsAllowed )
uasserted( BtreeBucket::dupKeyError( idx , keyLast ) );
}
keyLast = key;
try {
b->pushBack(loc, key, order, DiskLoc());
} catch( AssertionException& ) {
// no room
if ( key.objsize() > KeyMax ) {
problem() << "Btree::insert: key too large to index, skipping " << idx.indexNamespace().c_str() << ' ' << key.toString() << '\n';
}
else {
// bucket was full
newBucket();
b->pushBack(loc, key, order, DiskLoc());
}
}
}
void BtreeBuilder::buildNextLevel(DiskLoc loc) {
int levels = 1;
while( 1 ) {
if( loc.btree()->tempNext().isNull() ) {
// only 1 bucket at this level. we are done.
idx.head = loc;
break;
}
levels++;
DiskLoc upLoc = BtreeBucket::addBucket(idx);
DiskLoc upStart = upLoc;
BtreeBucket *up = upLoc.btreemod();
DiskLoc xloc = loc;
while( !xloc.isNull() ) {
BtreeBucket *x = xloc.btreemod();
BSONObj k;
DiskLoc r, rchild;
x->popBack(r,k,rchild);
assert( rchild.isNull() );
if( x->n == 0 )
log() << "warning: empty bucket on BtreeBuild " << k.toString() << endl;
try {
up->pushBack(r, k, order, xloc);
}
catch(AssertionException&) {
// current bucket full
DiskLoc n = BtreeBucket::addBucket(idx);
up->tempNext() = n;
upLoc = n;
up = upLoc.btreemod();
up->pushBack(r, k, order, xloc);
}
xloc = x->tempNext(); /* get next in chain at current level */
x->parent = upLoc;
}
loc = upStart;
}
log() << "TEMP " << "levels: " << levels << endl;
}
/* when all addKeys are done, we then build the higher levels of the tree */
void BtreeBuilder::commit() {
buildNextLevel(first);
committed = true;
}
BtreeBuilder::~BtreeBuilder() {
/* TODO: ROLLBACK CODE */
}
}

53
db/btree.h
View File

@ -78,6 +78,7 @@ namespace mongo {
/* this class is all about the storage management */
class BucketBasics {
friend class BtreeBuilder;
friend class KeyNode;
public:
void dumpTree(DiskLoc thisLoc, const BSONObj &order);
@ -86,10 +87,6 @@ namespace mongo {
int fullValidate(const DiskLoc& thisLoc, const BSONObj &order); /* traverses everything */
protected:
void modified(const DiskLoc& thisLoc);
DiskLoc& getChild(int pos) {
assert( pos >= 0 && pos <= n );
return pos == n ? nextChild : k(pos).prevChildBucket;
}
KeyNode keyNode(int i) const {
assert( i < n );
return KeyNode(*this, k(i));
@ -106,6 +103,8 @@ namespace mongo {
*/
bool basicInsert(const DiskLoc& thisLoc, int keypos, const DiskLoc& recordLoc, const BSONObj& key, const BSONObj &order);
void pushBack(const DiskLoc& recordLoc, BSONObj& key, const BSONObj &order, DiskLoc prevChild);
//void pushBack(const DiskLoc& recordLoc, BSONObj& key, const BSONObj &order, DiskLoc prevChild, DiskLoc nextChild);
void popBack(DiskLoc& recLoc, BSONObj& key, DiskLoc& rchild);
void _delKeyAtPos(int keypos); // low level version that doesn't deal with child ptrs.
/* !Packed means there is deleted fragment space within the bucket.
@ -114,7 +113,7 @@ namespace mongo {
*/
enum Flags { Packed=1 };
DiskLoc childForPos(int p) {
DiskLoc& childForPos(int p) {
return p == n ? nextChild : k(p).prevChildBucket;
}
@ -125,6 +124,12 @@ namespace mongo {
int _alloc(int bytes);
void truncateTo(int N, const BSONObj &order);
void markUnused(int keypos);
/* BtreeBuilder uses the parent var as a temp place to maintain a linked list chain.
we use tempNext() when we do that to be less confusing. (one might have written a union in C)
*/
DiskLoc& tempNext() { return parent; }
public:
DiskLoc parent;
@ -171,7 +176,7 @@ namespace mongo {
*/
bool exists(const IndexDetails& idx, DiskLoc thisLoc, const BSONObj& key, BSONObj order);
static DiskLoc addHead(IndexDetails&); /* start a new index off, empty */
static DiskLoc addBucket(IndexDetails&); /* start a new index off, empty */
static void renameIndexNamespace(const char *oldNs, const char *newNs);
@ -211,7 +216,9 @@ namespace mongo {
DiskLoc lChild, DiskLoc rChild, IndexDetails&);
bool find(const IndexDetails& idx, const BSONObj& key, DiskLoc recordLoc, const BSONObj &order, int& pos, bool assertIfDup);
static void findLargestKey(const DiskLoc& thisLoc, DiskLoc& largestLoc, int& largestKey);
string dupKeyError( const IndexDetails& idx , const BSONObj& key );
public:
// simply builds and returns a dup key error message string
static string dupKeyError( const IndexDetails& idx , const BSONObj& key );
};
class BtreeCursor : public Cursor {
@ -346,4 +353,36 @@ namespace mongo {
return head.btree()->exists(*this, head, key, keyPattern());
}
/* build btree from the bottom up */
/* _ TODO dropDups */
class BtreeBuilder {
bool dupsAllowed;
IndexDetails& idx;
unsigned long long n;
BSONObj keyLast;
BSONObj order;
bool committed;
DiskLoc cur, first;
BtreeBucket *b;
void newBucket();
void buildNextLevel(DiskLoc);
public:
~BtreeBuilder();
BtreeBuilder(bool _dupsAllowed, IndexDetails& _idx);
/* keys must be added in order */
void addKey(BSONObj& key, DiskLoc loc);
/* commit work. if not called, destructor will clean up partially completed work
(in case exception has happened).
*/
void commit();
unsigned long long getn() { return n; }
};
} // namespace mongo;

33
db/pdfile.cpp
View File

@ -1029,14 +1029,16 @@ assert( !eloc.isNull() );
}
}
int fastBuildIndex(const char *ns, NamespaceDetails *d, IndexDetails& idx, int idxNo) {
/* _ TODO dropDups */
unsigned long long fastBuildIndex(const char *ns, NamespaceDetails *d, IndexDetails& idx, int idxNo) {
bool dupsAllowed = !idx.unique();
bool dropDups = idx.dropDups();
BSONObj order = idx.keyPattern();
int n = 0;
/* get and sort all the keys ----- */
unsigned long long n = 0;
auto_ptr<Cursor> c = theDataFileMgr.findAll(ns);
BSONObjExternalSorter sorter;
BSONObjExternalSorter sorter(order);
int nkeys = 0;
while ( c->ok() ) {
BSONObj o = c->current();
@ -1057,24 +1059,23 @@ assert( !eloc.isNull() );
};
sorter.sort();
/* build index --- */
BtreeBuilder btBuilder(dupsAllowed, idx);
BSONObj keyLast;
BSONObjExternalSorter::Iterator i = sorter.iterator();
int m = 0;
while( i.more() ) {
BSONObjExternalSorter::Data d = i.next();
if( !dupsAllowed ) {
if( keyLast.woCompare(d.first) == 0 && m > 0 ) {
// duplicate
if( dropDups ) {
// ...
}
}
keyLast = d.first;
try {
btBuilder.addKey(d.first, d.second);
}
catch( AssertionException& ) {
massert("finish code!!!", !dupsAllowed); // todo dupsAllowed handle.
throw;
}
m++;
}
btBuilder.commit();
wassert( m == nkeys );
wassert( btBuilder.getn() == nkeys || dropDups );
return n;
}
@ -1113,10 +1114,10 @@ assert( !eloc.isNull() );
l.flush();
Timer t;
idx.head = BtreeBucket::addHead(idx);
idx.head = BtreeBucket::addBucket(idx);
int n = addExistingToIndex(ns.c_str(), d, idx, idxNo);
l << "done for " << n << " records";
l << "done for " << n << " records " << t.millis() / 1000.0 << "secs";
l << endl;
}

2
dbtests/btreetests.cpp
View File

@ -35,7 +35,7 @@ namespace BtreeTests {
BSONObj bobj = builder.done();
idx_.info =
theDataFileMgr.insert( ns(), bobj.objdata(), bobj.objsize() );
idx_.head = BtreeBucket::addHead( idx_ );
idx_.head = BtreeBucket::addBucket( idx_ );
}
~Base() {
// FIXME cleanup all btree buckets.