mirrors/mongodb
mirrors/mongodb
0
0
Fork 0
mirror of https://github.com/mongodb/mongo.git synced 2024-12-01 09:32:32 +01:00
Code Issues Releases Activity

more buildindex

Browse Source
This commit is contained in:
Dwight 2009-09-24 17:10:45 -04:00
parent 86f4a5480a
commit 29f0d36640
7 changed files with 91 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
db/btree.cpp
View File

@ -913,7 +913,7 @@ namespace mongo {
void BtreeBuilder::addKey(BSONObj& key, DiskLoc loc) {
if( n > 0 ) {
int cmp = keyLast.woCompare(key);
int cmp = keyLast.woCompare(key, order);
massert( "bad key order in BtreeBuilder - server internal error", cmp <= 0 );
if( cmp == 0 && !dupsAllowed )
uasserted( BtreeBucket::dupKeyError( idx , keyLast ) );
@ -988,7 +988,17 @@ namespace mongo {
}
BtreeBuilder::~BtreeBuilder() {
/* TODO: ROLLBACK CODE */
if( !committed ) {
log() << "TEMP ROLLING back partially built index space" << endl;
DiskLoc x = first;
while( !x.isNull() ) {
DiskLoc next = x.btree()->tempNext();
btreeStore->deleteRecord(idx.indexNamespace().c_str(), x);
x = next;
}
assert( idx.head.isNull() );
log() << "TEMP done rollback" << endl;
}
}
}

2
db/database.h
View File

@ -65,7 +65,7 @@ namespace mongo {
if( n >= RecCache::Base && n <= RecCache::Base+1000 )
massert("getFile(): bad file number - using recstore db w/nonrecstore db build?", false);
#endif
massert("getFile(): bad file number value (corrupt db?)", false);
massert("getFile(): bad file number value (corrupt db?): run repair", false);
}
DEV {
if ( n > 100 )

1
db/db.sln
View File

@ -21,7 +21,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shell", "shell", "{2CABB3B8
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{2B262D59-9DC7-4BF1-A431-1BD4966899A5}"
ProjectSection(SolutionItems) = preProject
..\tools\dump.cpp = ..\tools\dump.cpp
..\tools\importJSON.cpp = ..\tools\importJSON.cpp
EndProjectSection
EndProject

4
db/extsort.cpp
View File

@ -28,7 +28,7 @@
namespace mongo {
BSONObjExternalSorter::BSONObjExternalSorter( const BSONObj & order , long maxFileSize )
: _order( order ) , _maxFilesize( maxFileSize ) ,
: _order( order.getOwned() ) , _maxFilesize( maxFileSize ) ,
_map(0), _mapSizeSoFar(0), _largestObject(0), _sorted(0){
stringstream rootpath;
@ -77,7 +77,7 @@ namespace mongo {
_map = new InMemory( _order );
}
_map->insert( pair<BSONObj,DiskLoc>( o , loc ) );
_map->insert( pair<BSONObj,DiskLoc>( o.getOwned() , loc ) );
long size = o.objsize();
_mapSizeSoFar += size + sizeof( DiskLoc );

2
db/extsort.h
View File

@ -65,7 +65,7 @@ namespace mongo {
typedef set<Data,MyCmp> InMemory;
class Iterator {
class Iterator : boost::noncopyable {
public:
Iterator( BSONObjExternalSorter * sorter );

93
db/pdfile.cpp
View File

@ -1029,13 +1029,45 @@ assert( !eloc.isNull() );
}
}
void testSorting()
{
BSONObjBuilder b;
b.appendNull("");
BSONObj x = b.obj();
BSONObjExternalSorter sorter;
sorter.add(x, DiskLoc(3,7));
sorter.add(x, DiskLoc(4,7));
sorter.add(x, DiskLoc(2,7));
sorter.add(x, DiskLoc(1,7));
sorter.add(x, DiskLoc(3,77));
sorter.sort();
BSONObjExternalSorter::Iterator i = sorter.iterator();
while( i.more() ) {
BSONObjExternalSorter::Data d = i.next();
cout << d.second.toString() << endl;
cout << d.first.objsize() << endl;
cout<<"SORTER next:" << d.first.toString() << endl;
}
}
/* _ TODO dropDups
*/
unsigned long long fastBuildIndex(const char *ns, NamespaceDetails *d, IndexDetails& idx, int idxNo) {
testSorting();
cout << "\n\nBuildindex " << ns << " idxNo:" << idxNo;
cout << ' ' << idx.info.obj().toString() << endl;
bool dupsAllowed = !idx.unique();
bool dropDups = idx.dropDups();
BSONObj order = idx.keyPattern();
idx.head.Null();
/* get and sort all the keys ----- */
unsigned long long n = 0;
auto_ptr<Cursor> c = theDataFileMgr.findAll(ns);
@ -1051,6 +1083,7 @@ assert( !eloc.isNull() );
for ( BSONObjSetDefaultOrder::iterator i=keys.begin(); i != keys.end(); i++ ) {
if( ++k == 2 )
d->setIndexIsMultikey(idxNo);
cout<<"SORTER ADD " << i->toString() << ' ' << loc.toString() << endl;
sorter.add(*i, loc);
nkeys++;
}
@ -1060,36 +1093,56 @@ assert( !eloc.isNull() );
};
sorter.sort();
if( idxNo == 1 ) {
// TEMP!
BSONObjExternalSorter::Iterator i = sorter.iterator();
while( i.more() ) {
BSONObjExternalSorter::Data d = i.next();
cout << d.second.toString() << endl;
cout << d.first.objsize() << endl;
cout<<"SORTER next:" << d.first.toString() << endl;
}
cout << "stop here" << endl;
cout << "stop here" << endl;
}
list<DiskLoc> dupsToDrop;
/* build index --- */
BtreeBuilder btBuilder(dupsAllowed, idx);
BSONObj keyLast;
BSONObjExternalSorter::Iterator i = sorter.iterator();
while( i.more() ) {
BSONObjExternalSorter::Data d = i.next();
try {
btBuilder.addKey(d.first, d.second);
}
catch( AssertionException& ) {
if( !dupsAllowed ) {
if( dropDups ) {
/* we could queue these on disk, but normally there are very few dups, so instead we
keep in ram and have a limit.
*/
dupsToDrop.push_back(d.second);
uassert("too may dups on index build with dropDups=true", dupsToDrop.size() < 1000000 );
}
{
BtreeBuilder btBuilder(dupsAllowed, idx);
BSONObj keyLast;
BSONObjExternalSorter::Iterator i = sorter.iterator();
while( i.more() ) {
BSONObjExternalSorter::Data d = i.next();
cout<<"TEMP SORTER next " << d.first.toString() << endl;
//zzz
try {
btBuilder.addKey(d.first, d.second);
}
catch( AssertionException& ) {
if( !dupsAllowed ) {
if( dropDups ) {
/* we could queue these on disk, but normally there are very few dups, so instead we
keep in ram and have a limit.
*/
dupsToDrop.push_back(d.second);
uassert("too may dups on index build with dropDups=true", dupsToDrop.size() < 1000000 );
}
else
throw;
}
}
}
btBuilder.commit();
wassert( btBuilder.getn() == nkeys || dropDups );
}
for( list<DiskLoc>::iterator i = dupsToDrop.begin(); i != dupsToDrop.end(); i++ )
theDataFileMgr.deleteRecord( ns, i->rec(), *i, false, true );
btBuilder.commit();
wassert( btBuilder.getn() == nkeys || dropDups );
return n;
}
@ -1415,7 +1468,7 @@ assert( !eloc.isNull() );
int idxNo = tableToIndex->nIndexes;
IndexDetails& idx = tableToIndex->indexes[idxNo];
idx.info = loc;
tableToIndex->addingIndex(tabletoidxns.c_str(), idx); // clear transient info caches so they refresh
tableToIndex->addingIndex(tabletoidxns.c_str(), idx); // clear transient info caches so they refresh; increments nIndexes
try {
buildIndex(tabletoidxns, tableToIndex, idx, idxNo);
} catch( DBException& ) {

3
db/storage.h
View File

@ -70,7 +70,8 @@ namespace mongo {
assert(!isNull());
}
void setInvalid() {
fileNo = -2;
fileNo = -2;
ofs = 0;
}
bool isValid() const {
return fileNo != -2;