mirrors/mongodb
mirrors/mongodb
0
0
Fork 0
mirror of https://github.com/mongodb/mongo.git synced 2024-12-01 09:32:32 +01:00
Code Issues Releases Activity

make mongoimportjson not crash with small objects, some ProgressMeter enhancements SERVER-342

Browse Source
This commit is contained in:
Eliot Horowitz 2009-10-08 10:11:26 -04:00
parent 914bca4ff4
commit a54742b3de
2 changed files with 25 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
tools/importJSON.cpp
View File

@ -45,6 +45,7 @@ public:
int run(){ int run(){
string filename = getParam( "file" ); string filename = getParam( "file" );
long long fileSize = -1;
istream * in = &cin; istream * in = &cin;
@ -52,6 +53,7 @@ public:
if ( filename.size() > 0 && filename != "-" ){ if ( filename.size() > 0 && filename != "-" ){
in = &file; in = &file;
fileSize = file_size( filename );
} }
string ns; string ns;
@ -78,29 +80,35 @@ public:
time_t start = time(0); time_t start = time(0);
ProgressMeter pm( fileSize );
const int BUF_SIZE = 1024 * 1024 * 4; const int BUF_SIZE = 1024 * 1024 * 4;
char line[ (1024 * 1024 * 4) + 128]; char line[ (1024 * 1024 * 4) + 128];
while ( *in ){ while ( *in ){
in->getline( line , BUF_SIZE ); in->getline( line , BUF_SIZE );
int len = strlen( line );
if ( ! len )
break;
char * buf = line;
while( isspace( buf[0] ) ) buf++;
int len = strlen( buf );
if ( ! len )
continue;
if ( in->rdstate() == ios_base::eofbit )
break;
assert( in->rdstate() == 0 ); assert( in->rdstate() == 0 );
try { try {
BSONObj o = fromjson( line ); BSONObj o = fromjson( buf );
conn().insert( ns.c_str() , o ); conn().insert( ns.c_str() , o );
} }
catch ( MsgAssertionException& ma ){ catch ( MsgAssertionException& ma ){
cout << "exception:" << ma.toString() << endl; cout << "exception:" << ma.toString() << endl;
cout << line << endl; cout << buf << endl;
} }
if ( ++num % 10000 == 0 ){ num++;
cout << num << "\t" << ( num / ( time(0) - start ) ) << "/second" << endl; if ( pm.hit( len + 1 ) ){
cout << "\t\t\t" << num << "\t" << ( num / ( time(0) - start ) ) << "/second" << endl;
} }
} }

13
util/goodies.h
View File

@ -328,19 +328,22 @@ namespace mongo {
_done(0) , _hits(0) , _lastTime( (int) time(0) ){ _done(0) , _hits(0) , _lastTime( (int) time(0) ){
} }
void hit( int n = 1 ){ bool hit( int n = 1 ){
_done += n; _done += n;
_hits++; _hits++;
if ( _hits % _checkInterval ) if ( _hits % _checkInterval )
return; return false;
int t = (int) time(0); int t = (int) time(0);
if ( t - _lastTime < _secondsBetween ) if ( t - _lastTime < _secondsBetween )
return; return false;
int per = (int)( ( (double)_done * 100.0 ) / (double)_total ); if ( _total > 0 ){
cout << "\t\t" << _done << "/" << _total << "\t" << per << "%" << endl; int per = (int)( ( (double)_done * 100.0 ) / (double)_total );
cout << "\t\t" << _done << "/" << _total << "\t" << per << "%" << endl;
}
_lastTime = t; _lastTime = t;
return true;
} }
long long done(){ long long done(){