From 5e64eb609a75082e98cba551d2c24912528485a8 Mon Sep 17 00:00:00 2001
From: Eliot Horowitz <eliot@10gen.com>
Date: Thu, 29 Jan 2009 13:51:42 -0500
Subject: [PATCH] fixes and timing

---
 tools/importJSON.cpp | 52 ++++++++++++++++++++++----------------------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/tools/importJSON.cpp b/tools/importJSON.cpp
index 09e94acc305..42138a488dc 100644
--- a/tools/importJSON.cpp
+++ b/tools/importJSON.cpp
@@ -27,20 +27,10 @@
 
 #include <boost/program_options.hpp>
 
-#ifdef MODERN_BOOST
-#include <boost/iostreams/copy.hpp>
-#include <boost/iostreams/filtering_streambuf.hpp>
-#include <boost/iostreams/filter/gzip.hpp>
-
-using namespace boost::iostreams;
-#endif
-
 using namespace mongo;
 
-
 namespace po = boost::program_options;
 
-
 class ImportJSON : public Tool {
 public:
     ImportJSON() : Tool( "importjson" ){
@@ -58,18 +48,14 @@ public:
             return -1;
 
         }
+        
+        istream * in = &cin;
 
         ifstream file( filename.c_str() , ios_base::in | ios_base::binary);
-        
-#ifdef MODERN_BOOST
-        
-        filtering_streambuf<input> in;
-        in.push(gzip_decompressor());
-        in.push(file);
-        boost::iostreams::copy(in, cout);
-#else
-        istream & in = file;
-#endif
+            
+        if ( filename != "-" ){
+            in = &file;
+        }
         
         string ns = getNS();
 
@@ -78,20 +64,34 @@ public:
             _conn.dropCollection( ns.c_str() );
         }
         
-        while ( in ){
-            string line;
-            getline( in , line );
-            
-            if ( line.size() == 0 )
+        int num = 0;
+
+        time_t start = time(0);
+        
+        const int BUF_SIZE = 64000;
+        char line[64000 + 128];
+        while ( *in ){
+            in->getline( line , BUF_SIZE );
+
+            int len = strlen( line );
+            if ( ! len )
                 break;
             
+            assert( len < BUF_SIZE );
+                
+
             try {
                 BSONObj o = fromjson( line );
                 _conn.insert( ns.c_str() , o );
             }
             catch ( MsgAssertionException ma ){
                 cout << "exception:" << ma.toString() << endl;
-                cout << line << endl;
+                //cout << line << endl;
+            }
+            
+            if ( ++num % 10000 == 0 ){
+                cout << num << "\t" << ( num / ( time(0) - start ) ) << "/second" << endl;
+                
             }
         }