Merge branch 'master' of /data/gitroot/p/

db/btree.cpp

@@ -827,6 +827,8 @@ BtreeCursor::BtreeCursor(DiskLoc head, JSObj& k, int _direction, bool sm) :
 	checkUnused();
 }
 
+int zzz = 0;
+
 /* skip unused keys. */
 void BtreeCursor::checkUnused() {
 	int u = 0;
@@ -840,7 +842,7 @@ void BtreeCursor::checkUnused() {
 		bucket = b->advance(bucket, keyOfs, direction, "checkUnused");
 		u++;
 	}
-	if( u > 10 )
+	if( u > 10 && ++zzz % 16 == 0 )
 		cout << "btree unused skipped:" << u << endl;
 }
 

db/btree.h

@@ -168,6 +168,7 @@ public:
 	virtual DiskLoc currLoc() { return !bucket.isNull() ? _currKeyNode().recordLoc : DiskLoc(); }
 	virtual Record* _current() { return currLoc().rec(); }
 	virtual JSObj current() { return JSObj(_current()); }
+	virtual const char * toString() { return "BtreeCursor"; }
 
 private:
 	void checkUnused();

db/clientcursor.cpp

@@ -11,35 +11,40 @@
 #include "introspect.h"
 #include <time.h>
 
-/* TODO: FIX cleanup of clientCursors when hit the end. */
+/* TODO: FIX cleanup of clientCursors when hit the end. (ntoreturn insufficient) */
 
-typedef map<long long,ClientCursor*> CursorSet;
-map<DiskLoc, CursorSet> byLocation;
+typedef map<DiskLoc, ClientCursor*> DiskToCC;
+map<DiskLoc, ClientCursor*> byLocation;
+//HashTable<DiskLoc,ClientCursor*> byLocation(malloc(10000000), 10000000, "bylocation");
 
 CCMap clientCursors;
 
 class CursInspector : public SingleResultObjCursor { 
-	Cursor* clone() { return new CursInspector(*this); }
+	Cursor* clone() { 
+		return new CursInspector(); 
+	}
+//	Cursor* clone() { return new CursInspector(*this); }
 	void fill() { 
 		b.append("byLocation_size", byLocation.size());
 		b.append("clientCursors_size", clientCursors.size());
 
+cout << byLocation.size() << endl;
+
 		stringstream ss;
 		ss << '\n';
 		int x = 40;
-		map<DiskLoc,CursorSet>::iterator it = byLocation.begin();
+		DiskToCC::iterator it = byLocation.begin();
 		while( it != byLocation.end() ) {
 			DiskLoc dl = it->first;
-			CursorSet& v = it->second;
+			ClientCursor *cc = it->second;
 			ss << dl.toString() << " -> \n";
 
-			for( CursorSet::iterator j = v.begin(); j != v.end(); j++ ) {
-				ClientCursor *cc = j->second;
+			while( cc ) {
 				ss << "    cid:" << cc->cursorid << ' ' << cc->ns << " pos:" << cc->pos << " LL:" << cc->lastLoc.toString();
 				try { 
 					setClient(cc->ns.c_str());
 					Record *r = dl.rec();
-					ss << " lwh:" << hex << r->lengthWithHeaders << " nxt:" << r->nextOfs << " prv:" << r->prevOfs << dec;
+					ss << " lwh:" << hex << r->lengthWithHeaders << " nxt:" << r->nextOfs << " prv:" << r->prevOfs << dec << ' ' << cc->c->toString();
 					if( r->nextOfs >= 0 && r->nextOfs < 16 ) 
 						ss << " DELETED??? (!)";
 				}
@@ -47,6 +52,7 @@ class CursInspector : public SingleResultObjCursor {
 					ss << " EXCEPTION";
 				}
 				ss << "\n";
+				cc = cc->nextAtThisLocation;
 			}
 			if( --x <= 0 ) {
 				ss << "only first 40 shown\n" << endl;
@@ -67,58 +73,85 @@ void removedKey(const DiskLoc& btreeLoc, int keyPos) {
 
 /* must call this on a delete so we clean up the cursors. */
 void aboutToDelete(const DiskLoc& dl) { 
-	map<DiskLoc,CursorSet>::iterator it = byLocation.find(dl);
+	DiskToCC::iterator it = byLocation.find(dl);
+//	cout << "atd:" << dl.toString() << endl;
 	if( it != byLocation.end() ) {
-		CursorSet& cset = it->second;
+		ClientCursor *cc = it->second;
+		byLocation.erase(it);
 
-//		for( CursorSet::iterator j = cset.begin(); j != cset.end(); j++ ) {
-		CursorSet::iterator j = cset.begin();
-		while( j != cset.end() ) {
-			ClientCursor *cc = j->second;
-			assert( !cc->c->eof() );
-
-			// check if we are done before calling updateLocation which might
-			// delete cset
-			j++;
-			bool done = j == cset.end();
-
-			// dm testing attempt
+		assert( cc != 0 );
+		int z = 0;
+		while( cc ) { 
+			z++;
+//			cout << "cc: " << cc->ns << endl;
+			ClientCursor *nxt = cc->nextAtThisLocation;
+			cc->nextAtThisLocation = 0; // updateLocation will manipulate linked list ptrs, so clean that up first.
 			cc->c->checkLocation();
-
 			cc->c->advance();
+			cc->lastLoc.Null(); // so updateLocation doesn't try to remove, just to be faster -- we handled that.
 			cc->updateLocation();
-
-			if( done )
-				break;
+			cc = nxt;
 		}
+//		cout << "z:" << z << endl;
 	}
 }
 
 void ClientCursor::cleanupByLocation(DiskLoc loc, long long cursorid) { 
-	if( !loc.isNull() ) { 
-		map<DiskLoc, CursorSet>::iterator it = byLocation.find(loc);
-		if( it != byLocation.end() ) {
-			CursorSet& cset = it->second;
-			int n = cset.erase(cursorid);
-			wassert( n == 1 );
-			if( cset.size() == 0 )
-				byLocation.erase(it);
+	if( loc.isNull() )
+		return;
+
+	DiskToCC::iterator it = byLocation.find(loc);
+	if( it != byLocation.end() ) {
+		ClientCursor *first = it->second;
+		ClientCursor *cc = first;
+		ClientCursor *prev = 0;
+
+		while( 1 ) {
+			if( cc == 0 ) 
+				break;
+			if( cc->cursorid == cursorid ) { 
+				// found one to remove.
+				if( prev == 0 ) { 
+					if( cc->nextAtThisLocation )
+						byLocation[loc] = cc->nextAtThisLocation;
+					else
+						byLocation.erase(it);
+				}
+				else { 
+					prev->nextAtThisLocation = cc->nextAtThisLocation;
+				}
+				break;
+			}
+			cc = cc->nextAtThisLocation;
 		}
 	}
 }
 
 ClientCursor::~ClientCursor() {
+	assert( pos != -2 );
+
 	cleanupByLocation(lastLoc, cursorid);
 
+	assert( pos != -2 );
+
 	// defensive
 	lastLoc.Null();
 	cursorid = -1; 
 	pos = -2;
+	nextAtThisLocation = 0;
 }
 
 // note this doesn't set lastLoc -- caller should.
 void ClientCursor::addToByLocation(DiskLoc cl) { 
-	byLocation[cl][cursorid] = this;
+//if( 1 )
+//return;
+//TEMP!
+
+	assert( nextAtThisLocation == 0 );
+
+	DiskToCC::iterator j = byLocation.find(cl);
+	nextAtThisLocation = j == byLocation.end() ? 0 : j->second;
+	byLocation[cl] = this;
 }
 
 void ClientCursor::updateLocation() {
@@ -136,7 +169,9 @@ void ClientCursor::updateLocation() {
 	c->noteLocation();
 }
 
-/* report to us that a new clientcursor exists so we can track it. */
+/* report to us that a new clientcursor exists so we can track it. 
+   note you still must call updateLocation (which likely should be changed)
+*/
 void ClientCursor::add(ClientCursor* cc) {
 	clientCursors[cc->cursorid] = cc;
 }
@@ -158,10 +193,9 @@ long long allocCursorId() {
 	long long x;
 	while( 1 ) {
 		x = (((long long)rand()) << 32);
-		x = x | time(0);
+		x = x | (int) curTimeMillis() | 0x80000000; // last or to w make sure not zero
 		if( clientCursors.count(x) == 0 )
 			break;
 	}
 	return x;
 }
-

db/db.cpp

@@ -272,7 +272,7 @@ public:
 };
 
 void listen(int port) { 
-	cout << "db version: 25feb08.1 unindex fixes" << endl;
+	cout << "db version: 27feb08.1 latent cursor fixes" << endl;
 	pdfileInit();
 	testTheDb();
 	cout << curTimeMillis() % 10000 << " waiting for connections...\n" << endl;

db/db.vcproj

@@ -21,6 +21,7 @@
 			OutputDirectory="$(SolutionDir)$(ConfigurationName)"
 			IntermediateDirectory="$(ConfigurationName)"
 			ConfigurationType="1"
+			UseOfMFC="0"
 			UseOfATL="0"
 			CharacterSet="1"
 			>

db/introspect.h

@@ -32,5 +32,7 @@ public:
 		return false;
 	}
 
+	virtual const char * toString() { return "SingleResultObjCursor"; }
+
 };
 

db/jsobj.h

@@ -385,6 +385,8 @@ public:
 
 	bool matches(JSObj& j, bool *deep = 0);
 
+	int getN() { return n; }
+
 private:
 	int valuesMatch(Element& l, Element& r, int op);
 

db/pdfile.h

@@ -222,6 +222,8 @@ public:
 	/* called before query getmore block is iterated */
 	virtual void checkLocation() { } 
 
+	virtual const char * toString() { return "abstract?"; }
+
 	/* used for multikey index traversal to avoid sending back dups. see JSMatcher::matches() */
 	set<DiskLoc> dups;
 	bool dup(DiskLoc loc) {
@@ -258,6 +260,8 @@ public:
 	BasicCursor(DiskLoc dl) : curr(dl) { }
 	BasicCursor() { }
 
+	virtual const char * toString() { return "BasicCursor"; }
+
 	DiskLoc curr;
 };
 

db/query.cpp

@@ -486,6 +486,8 @@ QueryResult* runQuery(const char *ns, int ntoskip, int ntoreturn, JSObj jsobj,
 			query = jsobj;
 
 		auto_ptr<JSMatcher> matcher(new JSMatcher(query));
+JSMatcher &debug1 = *matcher;
+assert( debug1.getN() < 5000 );
 
 		int nscanned = 0;
 		auto_ptr<Cursor> c = getSpecialCursor(ns);
@@ -508,6 +510,9 @@ QueryResult* runQuery(const char *ns, int ntoskip, int ntoreturn, JSObj jsobj,
 				nscanned++;
 				bool deep;
 
+JSMatcher &debug = *matcher;
+assert( debug.getN() < 5000 );
+
 				if( !matcher->matches(js, &deep) ) {
 					if( c->tempStopOnMiss() )
 						break;

db/query.h

@@ -86,8 +86,9 @@ extern CCMap clientCursors; /* cursorid -> ClientCursor */
 */
 class Cursor;
 class ClientCursor {
+	friend class CursInspector;
 public:
-	ClientCursor() { cursorid=0; pos=0; }
+	ClientCursor() { cursorid=0; pos=0; nextAtThisLocation=0; }
 	~ClientCursor();
 	long long cursorid;
 	string ns;
@@ -122,6 +123,8 @@ public:
 private:
 	void addToByLocation(DiskLoc cl);
 	static void cleanupByLocation(DiskLoc loc, long long cursorid);
+public:
+	ClientCursor *nextAtThisLocation;
 };
 
 long long allocCursorId();

db/testdb.js

@@ -72,6 +72,8 @@ function bigIndexTest() {
 }
 
 function runall() { 
+    runcursors();
+
     runquick();
 
     print("bigindextest stuff:");
@@ -137,13 +139,36 @@ function testdups2() {
 }
 
 function runcursors() {
-    for( i = 0; i < 50; i++ )
-	t.cur.save( { name:"ABC", k:/asfd/, a:i } );
-
-    c = t.cur.find().limit(2);
-    print(c[0].name);
-
+    print("initial remove");
     t.cur.remove({});
+    t.cur.findOne();
+    print(" done");
+
+    print( tojson( connect("intr").cursors.find() ) );
+
+    print("insert");
+
+    for( i = 0; i < 50000; i++ )
+	t.cur.save( { name:"ABC", k:/asfd/, a:i, 
+		    lng:"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+		    lng1:"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+		    lng2:"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"} );
+
+    for( i = 0; i < 100; i++ ) {
+	c = t.cur.find().limit(2);
+	print(c[0].name);
+	t.cur.find({name:"ABC"}).limit(3)[0];
+    }
+
+    print( tojson( connect("intr").cursors.find() ) );
+
+    print("Remove...");
+    t.cur.remove({});
+    t.cur.findOne();
+    print(" done");
+
+    print( tojson( connect("intr").cursors.find() ) );
+
     print(" end runcursors");
 }
 

util/builder.h

@@ -9,7 +9,7 @@
 class BufBuilder {
 public:
 	BufBuilder(int initsize = 512) : size(initsize) { 
-		data = (char *) malloc(size);
+		data = (char *) malloc(size); assert(data);
 		l = 0;
 	}
 	~BufBuilder() { kill(); }