towards compact

db/compact.cpp

@@ -26,6 +26,8 @@
 #include "commands.h"
 #include "curop-inl.h"
 #include "background.h"
+#include "extsort.h"
+#include "compact.h"
 #include "../util/concurrency/task.h"
 
 namespace mongo {
@@ -36,7 +38,10 @@ namespace mongo {
     DiskLoc allocateSpaceForANewRecord(const char *ns, NamespaceDetails *d, int lenWHdr);
     void freeExtents(DiskLoc firstExt, DiskLoc lastExt);
 
-    void compactExtent(const char *ns, NamespaceDetails *d, DiskLoc ext, int n) { 
+    void compactExtent(const char *ns, NamespaceDetails *d, DiskLoc ext, int n,
+                const scoped_array<IndexSpec> &indexSpecs,
+                scoped_array<SortPhaseOne>& phase1, int nidx)
+    {
         log() << "compact extent #" << n << endl;
         Extent *e = ext.ext();
         e->assertOk();
@@ -68,6 +73,7 @@ namespace mongo {
                 L = recOld->nextInExtent(L);
                 nrecs++;
                 BSONObj objOld(recOld);
+
                 unsigned sz = objOld.objsize();
                 unsigned lenWHdr = sz + Record::HeaderSize;
                 totalSize += lenWHdr;
@@ -79,6 +85,23 @@ namespace mongo {
                 addRecordToRecListInExtent(recNew, loc);
                 memcpy(recNew->data, objOld.objdata(), sz);
 
+                {
+                    // extract keys for all indexes we will be rebuilding
+                    for( int x = 0; x < nidx; x++ ) { 
+                        BSONObjSetDefaultOrder keys;
+                        indexSpecs[x].getKeys(objOld, keys);
+                        SortPhaseOne& p1 = phase1[x];
+                        int k = 0;
+                        for ( BSONObjSetDefaultOrder::iterator i=keys.begin(); i != keys.end(); i++ ) {
+                            if( ++k == 2 ) {
+                                p1.multi = true;
+                            }
+                            p1.sorter->add(*i, loc);
+                            p1.nkeys++;
+                        }                    
+                    }
+                }
+
                 if( L.isNull() ) { 
                     // we just did the very last record from the old extent.  it's still pointed to 
                     // by the old extent ext, but that will be fixed below after this loop
@@ -107,6 +130,8 @@ namespace mongo {
         // drop this extent
     }
 
+    extern SortPhaseOne *precalced;
+
     bool _compact(const char *ns, NamespaceDetails *d, string& errmsg) { 
         //int les = d->lastExtentSize;
 
@@ -121,18 +146,25 @@ namespace mongo {
         // same data, but might perform a little different after compact?
         NamespaceDetailsTransient::get_w(ns).clearQueryCache();
 
-        list<BSONObj> indexes;
+        int nidx = d->nIndexes;
+        scoped_array<IndexSpec> indexSpecs( new IndexSpec[nidx] );
+        scoped_array<SortPhaseOne> phase1( new SortPhaseOne[nidx] );
         {
             NamespaceDetails::IndexIterator ii = d->ii(); 
+            int x = 0;
             while( ii.more() ) { 
                 BSONObjBuilder b;
                 BSONObj::iterator i(ii.next().info.obj());
                 while( i.more() ) { 
                     BSONElement e = i.next();
-                    if( strcmp(e.fieldName(), "v") != 0 && strcmp(e.fieldName(), "background") != 0 )
+                    if( strcmp(e.fieldName(), "v") != 0 && strcmp(e.fieldName(), "background") != 0 ) {
                         b.append(e);
+                    }
                 }
-                indexes.push_back( b.obj() );
+                BSONObj o = b.obj().getOwned();
+                phase1[x].sorter.reset( new BSONObjExternalSorter( o.getObjectField("key") ) );
+                phase1[x].sorter->hintNumObjects( d->stats.nrecords );
+                indexSpecs[x++].reset(o);
             }
         }
 
@@ -156,7 +188,7 @@ namespace mongo {
 
         int n = 0;
         for( set<DiskLoc>::iterator i = extents.begin(); i != extents.end(); i++ ) { 
-            compactExtent(ns, d, *i, n++);
+            compactExtent(ns, d, *i, n++, indexSpecs, phase1, nidx);
         }
 
         assert( d->firstExtent.ext()->xprev.isNull() );
@@ -164,9 +196,18 @@ namespace mongo {
         // build indexes
         NamespaceString s(ns);
         string si = s.db + ".system.indexes";
-        for( list<BSONObj>::iterator i = indexes.begin(); i != indexes.end(); i++ ) {
-            log() << "compact create index " << (*i)["key"].Obj().toString() << endl;
-            theDataFileMgr.insert(si.c_str(), i->objdata(), i->objsize());
+        for( int i = 0; i < nidx; i++ ) {
+            BSONObj info = indexSpecs[i].info;
+            log() << "compact create index " << info["key"].Obj().toString() << endl;
+            try {
+                precalced = &phase1[i];
+                theDataFileMgr.insert(si.c_str(), info.objdata(), info.objsize());
+            }
+            catch(...) { 
+                precalced = 0;
+                throw;
+            }
+            precalced = 0;
         }
 
         return true;

db/compact.h

@@ -0,0 +1,36 @@
+// compact.h
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+namespace mongo {
+
+    /** for bottom up fastbuildindex (where we presort keys) */
+    struct SortPhaseOne { 
+        SortPhaseOne() { 
+            n = 0;
+            nkeys = 0;
+            multi = false;
+        }
+        shared_ptr<BSONObjExternalSorter> sorter;
+        unsigned long long n; // # of records
+        unsigned long long nkeys;
+        bool multi; // multikey index
+    };
+
+}

db/index.cpp

@@ -276,14 +276,13 @@ namespace mongo {
         return true;
     }
 
-
     void IndexSpec::reset( const IndexDetails * details ) {
         _details = details;
         reset( details->info );
     }
 
-    void IndexSpec::reset( const DiskLoc& loc ) {
-        info = loc.obj();
+    void IndexSpec::reset( const BSONObj& _info ) {
+        info = _info;
         keyPattern = info["key"].embeddedObjectUserCheck();
         if ( keyPattern.objsize() == 0 ) {
             out() << info.toString() << endl;

db/indexkey.h

@@ -135,7 +135,8 @@ namespace mongo {
             reset( loc );
         }
 
-        void reset( const DiskLoc& loc );
+        void reset( const BSONObj& info );
+        void reset( const DiskLoc& infoLoc ) { reset(infoLoc.obj()); }
         void reset( const IndexDetails * details );
 
         void getKeys( const BSONObj &obj, BSONObjSetDefaultOrder &keys ) const;

db/pdfile.cpp

@@ -41,6 +41,7 @@ _ disallow system* manipulations from the database.
 #include "extsort.h"
 #include "curop-inl.h"
 #include "background.h"
+#include "compact.h"
 
 namespace mongo {
 
@@ -1123,6 +1124,8 @@ namespace mongo {
         }
     }
 
+    SortPhaseOne *precalced = 0;
+
     // throws DBException
     unsigned long long fastBuildIndex(const char *ns, NamespaceDetails *d, IndexDetails& idx, int idxNo) {
         CurOp * op = cc().curop();
@@ -1140,39 +1143,49 @@ namespace mongo {
         if ( logLevel > 1 ) printMemInfo( "before index start" );
 
         /* get and sort all the keys ----- */
-        unsigned long long n = 0;
-        shared_ptr<Cursor> c = theDataFileMgr.findAll(ns);
-        BSONObjExternalSorter sorter(order);
-        sorter.hintNumObjects( d->stats.nrecords );
-        unsigned long long nkeys = 0;
         ProgressMeterHolder pm( op->setMessage( "index: (1/3) external sort" , d->stats.nrecords , 10 ) );
-        while ( c->ok() ) {
-            BSONObj o = c->current();
-            DiskLoc loc = c->currLoc();
+        SortPhaseOne _ours;
+        SortPhaseOne *phase1 = precalced;
+        if( phase1 == 0 ) {
+            phase1 = &_ours;
+            SortPhaseOne& p1 = *phase1;
+            shared_ptr<Cursor> c = theDataFileMgr.findAll(ns);
+            p1.sorter.reset( new BSONObjExternalSorter(order) );
+            p1.sorter->hintNumObjects( d->stats.nrecords );
+            BSONObjExternalSorter& sorter = *p1.sorter;
+            const IndexSpec& spec = idx.getSpec();
+            while ( c->ok() ) {
+                BSONObj o = c->current();
+                DiskLoc loc = c->currLoc();
 
-            BSONObjSetDefaultOrder keys;
-            idx.getKeysFromObject(o, keys);
-            int k = 0;
-            for ( BSONObjSetDefaultOrder::iterator i=keys.begin(); i != keys.end(); i++ ) {
-                if( ++k == 2 ) {
-                    d->setIndexIsMultikey(idxNo);
+                BSONObjSetDefaultOrder keys;
+                spec.getKeys(o, keys);
+                int k = 0;
+                for ( BSONObjSetDefaultOrder::iterator i=keys.begin(); i != keys.end(); i++ ) {
+                    if( ++k == 2 ) {
+                        p1.multi = true;
+                    }
+                    sorter.add(*i, loc);
+                    p1.nkeys++;
                 }
-                sorter.add(*i, loc);
-                nkeys++;
-            }
 
-            c->advance();
-            n++;
-            pm.hit();
-            if ( logLevel > 1 && n % 10000 == 0 ) {
-                printMemInfo( "\t iterating objects" );
-            }
-
-        };
+                c->advance();
+                p1.n++;
+                pm.hit();
+                if ( logLevel > 1 && p1.n % 10000 == 0 ) {
+                    printMemInfo( "\t iterating objects" );
+                }
+            };
+        }
         pm.finished();
 
+        BSONObjExternalSorter& sorter = *(phase1->sorter);
+
+        if( phase1->multi )
+            d->setIndexIsMultikey(idxNo);
+
         if ( logLevel > 1 ) printMemInfo( "before final sort" );
-        sorter.sort();
+        phase1->sorter->sort();
         if ( logLevel > 1 ) printMemInfo( "after final sort" );
 
         log(t.seconds() > 5 ? 0 : 1) << "\t external sort used : " << sorter.numFiles() << " files " << " in " << t.seconds() << " secs" << endl;
@@ -1184,7 +1197,7 @@ namespace mongo {
             BtreeBuilder btBuilder(dupsAllowed, idx);
             BSONObj keyLast;
             auto_ptr<BSONObjExternalSorter::Iterator> i = sorter.iterator();
-            assert( pm == op->setMessage( "index: (2/3) btree bottom up" , nkeys , 10 ) );
+            assert( pm == op->setMessage( "index: (2/3) btree bottom up" , phase1->nkeys , 10 ) );
             while( i->more() ) {
                 RARELY killCurrentOp.checkForInterrupt();
                 BSONObjExternalSorter::Data d = i->next();
@@ -1216,7 +1229,7 @@ namespace mongo {
             op->setMessage( "index: (3/3) btree-middle" );
             log(t.seconds() > 10 ? 0 : 1 ) << "\t done building bottom layer, going to commit" << endl;
             btBuilder.commit();
-            if ( btBuilder.getn() != nkeys && ! dropDups ) {
+            if ( btBuilder.getn() != phase1->nkeys && ! dropDups ) {
                 warning() << "not all entries were added to the index, probably some keys were too large" << endl;
             }
         }
@@ -1228,7 +1241,7 @@ namespace mongo {
             getDur().commitIfNeeded();
         }
 
-        return n;
+        return phase1->n;
     }
 
     class BackgroundIndexBuildJob : public BackgroundOperation {

jstests/compact.js

@@ -4,7 +4,8 @@ db.dropDatabase();
 
 t = db.compacttest;
 t.drop();
-t.insert({});
+t.insert({ x: 3 });
+t.ensureIndex({ x: 1 });
 
 print("1");
 
@@ -16,7 +17,7 @@ var v = t.validate(true);
 assert(v.ok);
 assert(v.extentCount == 1);
 assert(v.deletedCount == 1);
-assert(t.getIndexes().length == 1);
+assert(t.getIndexes().length == 2);
 
 print("2");
 
@@ -27,4 +28,4 @@ assert(t.count() == 0);
 v = t.validate(true);
 assert(v.ok);
 assert(v.extentCount == 1);
-assert(t.getIndexes().length == 1);
+assert(t.getIndexes().length == 2);