Revert "Revert "do preallocation after initial allocation too, make file allocator interface more flexible""

This reverts commit 668cf57c52.

db/database.h

@@ -108,9 +108,15 @@ namespace mongo {
             int n = (int) files.size();
             MongoDataFile *ret = getFile( n, sizeNeeded );
             if ( preallocateNextFile )
-                getFile( n + 1, 0, true );
+                preallocateAFile();
             return ret;
         }
+        
+        // ok to call multiple times
+        void preallocateAFile() {
+            int n = (int) files.size();
+            getFile( n, 0, true );            
+        }
 
         MongoDataFile* suitableFile( int sizeNeeded ) {
             MongoDataFile* f = newestFile();

db/pdfile.cpp

@@ -143,13 +143,18 @@ namespace mongo {
             for ( int i = 0; i < nExtents; ++i ) {
                 database->suitableFile((int) size)->allocExtent( ns, (int) size, newCapped );
             }
-        } else
+        } else {
             while ( size > 0 ) {
                 int max = MongoDataFile::maxSize() - MDFHeader::headerSize();
                 int desiredExtentSize = (int) (size > max ? max : size);
                 Extent *e = database->suitableFile( desiredExtentSize )->allocExtent( ns, desiredExtentSize, newCapped );
                 size -= e->length;
             }
+            if ( !newCapped ) {
+                // ok to call this multiple times
+                database->preallocateAFile();
+            }
+        }
 
         NamespaceDetails *d = nsdetails(ns);
         assert(d);
@@ -233,8 +238,6 @@ namespace mongo {
 
         if ( preallocateOnly ) {
 #if !defined(_WIN32)
-            // if file exists, update 'size' to match existing file size.
-            MemoryMappedFile::updateLength( filename, size );
             theFileAllocator().requestAllocation( filename, size );
 #endif
             return;

util/file_allocator.h

@@ -25,45 +25,54 @@ namespace mongo {
 
     // Handles allocation of contiguous files on disk.
     class FileAllocator {
-        // The public functions may not be called concurrently.  If
+        // The public functions may not be called concurrently.  The allocation
-        // allocateAsap() is called for a file after requestAllocation(), the
+        // functions may be called multiple times per file, but only the first
-        // sizes in each call must be the same.
+        // size specified per file will be used.
     public:
         void start() {
             Runner r( *this );
             boost::thread t( r );
         }
-        // May be called if file exists, but may not be called more than once
+        // May be called if file exists. If file exists, or its allocation has
-        // for a file.
+        // been requested, size is updated to match existing file size.
-        void requestAllocation( const string &name, int size ) {
+        void requestAllocation( const string &name, int &size ) {
-            if ( boost::filesystem::exists( name ) )
-                return;
             {
                 boostlock lk( pendingMutex_ );
-                pending_.push_back( make_pair( name, size ) );
+                int oldSize = prevSize( name );
+                if ( oldSize != -1 ) {
+                    size = oldSize;
+                    return;
+                }
+                pending_.push_back( name );
+                pendingSize_[ name ] = size;
             }
             pendingUpdated_.notify_all();
         }
-        // Returns when file has been allocated.
+        // Returns when file has been allocated.  If file exists, size is
+        // updated to match existing file size.
         void allocateAsap( const string &name, int size ) {
-            pair< string, int > spec( name, size );
             {
                 boostlock lk( pendingMutex_ );
-                if ( allocated( name ) )
+                int oldSize = prevSize( name );
-                    return;
+                if ( oldSize != -1 ) {
+                    size = oldSize;
+                    if ( !inProgress( name ) )
+                        return;
+                }
+                pendingSize_[ name ] = size;
                 if ( pending_.size() == 0 )
-                    pending_.push_back( spec );
+                    pending_.push_back( name );
-                else if ( pending_.front() != spec ) {
+                else if ( pending_.front() != name ) {
-                    pending_.remove( spec );
+                    pending_.remove( name );
-                    list< pair< string, int > >::iterator i = pending_.begin();
+                    list< string >::iterator i = pending_.begin();
                     ++i;
-                    pending_.insert( i, spec );
+                    pending_.insert( i, name );
                 }
             }
             pendingUpdated_.notify_all();
             boostlock lk( pendingMutex_ );
             while( 1 ) {
-                if ( allocated( name ) ) {
+                if ( !inProgress( name ) ) {
                     return;
                 }
                 pendingUpdated_.wait( lk );                    
@@ -80,19 +89,28 @@ namespace mongo {
         }
         
     private:
-        // caller must hold pendingMutex_ lock
+        // caller must hold pendingMutex_ lock.  Returns size if allocated or 
-        bool allocated( const string &name ) const {
+        // allocation requested, -1 otherwise.
-            if ( !boost::filesystem::exists( name ) )
+        int prevSize( const string &name ) const {
-                return false;
+            if ( pendingSize_.count( name ) > 0 )
-            for( list< pair< string, int > >::const_iterator i = pending_.begin(); i != pending_.end(); ++i )
+                return pendingSize_[ name ];
-                if ( i->first == name )
+            if ( boost::filesystem::exists( name ) )
-                    return false;
+                return boost::filesystem::file_size( name );
-            return true;
+            return -1;
+        }
+         
+        // caller must hold pendingMutex_ lock.
+        bool inProgress( const string &name ) const {
+            for( list< string >::const_iterator i = pending_.begin(); i != pending_.end(); ++i )
+                if ( *i == name )
+                    return true;
+            return false;
         }
 
         mutable boost::mutex pendingMutex_;
         mutable boost::condition_variable pendingUpdated_;
-        list< pair< string, int > > pending_;
+        list< string > pending_;
+        mutable map< string, int > pendingSize_;
         
         struct Runner {
             Runner( FileAllocator &allocator ) : a_( allocator ) {}
@@ -111,8 +129,8 @@ namespace mongo {
                             boostlock lk( a_.pendingMutex_ );
                             if ( a_.pending_.size() == 0 )
                                 break;
-                            name = a_.pending_.front().first;
+                            name = a_.pending_.front();
-                            size = a_.pending_.front().second;
+                            size = a_.pendingSize_[ name ];
                         }
                         try {
                             int fd = open(name.c_str(), O_CREAT | O_RDWR | O_NOATIME, S_IRUSR | S_IWUSR);
@@ -157,6 +175,7 @@ namespace mongo {
                         
                         {
                             boostlock lk( a_.pendingMutex_ );
+                            a_.pendingSize_.erase( name );
                             a_.pending_.pop_front();
                         }
                         a_.pendingUpdated_.notify_all();

util/mmap_posix.cpp

@@ -51,11 +51,10 @@ namespace mongo {
 #endif
 
     void* MemoryMappedFile::map(const char *filename, int length) {
-        updateLength( filename, length );
+        // length may be updated by callee.
-        len = length;
-
         theFileAllocator().allocateAsap( filename, length );
-
+        len = length;
+        
         fd = open(filename, O_RDWR | O_NOATIME);
         if ( fd <= 0 ) {
             out() << "couldn't open " << filename << ' ' << errno << endl;