mirrors/mongodb
mirrors/mongodb
0
0
Fork 0
mirror of https://github.com/mongodb/mongo.git synced 2024-12-01 09:32:32 +01:00
Code Issues Releases Activity

recstore: support for multiple directories

Browse Source 
plus bug fix
This commit is contained in:
Dwight 2009-03-23 16:25:03 -04:00
parent 3426f0e8c7
commit ea2df30205
2 changed files with 75 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
db/reccache.cpp
View File

@ -1,9 +1,13 @@
// storage.cpp
#include "stdafx.h"
int aaa;
#include "pdfile.h"
#include "reccache.h"
#include "rec.h"
#include "db.h"
namespace mongo {
@ -32,6 +36,40 @@ void recCacheCloseAll() {
int ndirtywritten;
inline static string escape(const char *ns) {
char buf[256];
char *p = buf;
while( 1 ) {
if( *ns == '$' ) *p = '~';
else
*p = *ns;
if( *ns == 0 )
break;
p++; ns++;
}
assert( p - buf < (int) sizeof(buf) );
return buf;
}
inline static string unescape(const char *ns) {
char buf[256];
char *p = buf;
while( 1 ) {
if( *ns == '~' ) *p = '$';
else
*p = *ns;
if( *ns == 0 )
break;
p++; ns++;
}
assert( p - buf < (int) sizeof(buf) );
return buf;
}
string RecCache::directory() {
return database->path;
}
/* filename format is
<n>-<ns>.idx
@ -54,7 +92,7 @@ BasicRecStore* RecCache::_initStore(string fname) {
const char *p = rest.c_str();
const char *q = strstr(p, ".idx");
assert( q );
string ns(p, q-p);
string escaped_ns(p, q-p);
// arbitrary limit. if you are hitting, we should use fewer files and put multiple
// indexes in a single file (which is easy to do)
@ -64,12 +102,13 @@ BasicRecStore* RecCache::_initStore(string fname) {
stores.resize(n+1);
assert( stores[n] == 0 );
BasicRecStore *rs = new BasicRecStore(n);
path pf(dbpath);
path pf(directory());
pf /= fname;
string full = pf.string();
rs->init(full.c_str(), recsize);
stores[n] = rs;
storesByNs[ns] = rs;
string ns = unescape(escaped_ns.c_str());
storesByNsKey[mknskey(ns.c_str())] = rs;
return rs;
}
@ -82,7 +121,7 @@ BasicRecStore* RecCache::initStore(int n) {
}
/* this will be slow if there are thousands of files */
path dir(dbpath);
path dir(directory());
directory_iterator end;
try {
directory_iterator i(dir);
@ -101,7 +140,7 @@ BasicRecStore* RecCache::initStore(int n) {
throw;
}
catch (...) {
string s = string("i/o error looking for .idx file in ") + dbpath;
string s = string("i/o error looking for .idx file in ") + directory();
massert(s, false);
}
stringstream ss;
@ -121,13 +160,12 @@ string RecCache::findStoreFilename(const char *_ns, bool& found) {
{
stringstream ss;
ss << '-';
ss << _ns;
assert( strchr(_ns, '$') == 0); // $ not good for filenames
ss << escape(_ns);
ss << ".idx";
namefrag = ss.str();
}
path dir(dbpath);
path dir(directory());
directory_iterator end;
int nmax = -1;
try {
@ -150,7 +188,7 @@ string RecCache::findStoreFilename(const char *_ns, bool& found) {
}
}
catch (...) {
string s = string("i/o error looking for .idx file in ") + dbpath;
string s = string("i/o error looking for .idx file in ") + directory();
massert(s, false);
}
@ -161,7 +199,7 @@ string RecCache::findStoreFilename(const char *_ns, bool& found) {
return ss.str();
}
void RecCache::initStoreByNs(const char *_ns) {
void RecCache::initStoreByNs(const char *_ns, const string& nskey) {
bool found;
string fn = findStoreFilename(_ns, found);
_initStore(fn);
@ -275,16 +313,17 @@ void RecCache::dump() {
}
void RecCache::closeStore(BasicRecStore *rs) {
int n = rs->fileNumber + Base;
for( set<DiskLoc>::iterator i = dirtyl.begin(); i != dirtyl.end(); ) {
DiskLoc k = *i++;
if( k.a() == rs->fileNumber )
if( k.a() == n )
dirtyl.erase(k);
}
for( map<DiskLoc,Node*>::iterator i = m.begin(); i != m.end(); ) {
DiskLoc k = i->first;
i++;
if( k.a() == rs->fileNumber )
if( k.a() == n )
m.erase(k);
}
@ -301,21 +340,7 @@ void RecCache::drop(const char *_ns) {
// todo: test with a non clean shutdown file
boostlock lk(rcmutex);
char buf[256];
{
const char *ns = _ns;
char *p = buf;
while( 1 ) {
if( *ns == '$' ) *p = '_';
else
*p = *ns;
if( *ns == 0 )
break;
p++; ns++;
}
assert( p - buf < (int) sizeof(buf) );
}
BasicRecStore *&rs = storesByNs[buf];
BasicRecStore *&rs = storesByNsKey[mknskey(_ns)];
string fname;
if( rs ) {
fname = rs->filename;
@ -324,16 +349,18 @@ void RecCache::drop(const char *_ns) {
}
else {
bool found;
fname = findStoreFilename(buf, found);
fname = findStoreFilename(_ns, found);
if( !found ) {
log() << "RecCache::drop: no idx file found for " << _ns << endl;
return;
}
path pf(dbpath);
path pf(directory());
pf /= fname;
fname = pf.string();
}
try {
try {
if( !boost::filesystem::exists(fname) )
log() << "RecCache::drop: can't find file to remove " << fname << endl;
boost::filesystem::remove(fname);
}
catch(...) {

39
db/reccache.h
View File

@ -31,51 +31,48 @@ class RecCache {
char *data;
DiskLoc loc;
bool dirty;
Node *older, *newer;
Node *older, *newer; // lru
};
boost::mutex rcmutex; // mainly to coordinate with the lazy writer thread
unsigned recsize;
map<DiskLoc, Node*> m;
map<DiskLoc, Node*> m; // the cache
Node *newest, *oldest;
unsigned nnodes;
set<DiskLoc> dirtyl;
vector<BasicRecStore*> stores;
map<string, BasicRecStore*> storesByNs;
vector<BasicRecStore*> stores; // DiskLoc::a() indicates the index into this vector
map<string, BasicRecStore*> storesByNsKey; // nskey -> BasicRecStore*
public:
enum { Base = 10000 };
private:
BasicRecStore* _initStore(string fname);
BasicRecStore* initStore(int n);
string findStoreFilename(const char *_ns, bool& found);
void initStoreByNs(const char *escaped_ns);
void initStoreByNs(const char *ns, const string& nskey);
void closeStore(BasicRecStore *rs);
static string directory();
static string mknskey(const char *ns) {
return directory() + ns;
}
/* get the right file for a given diskloc */
BasicRecStore& store(DiskLoc& d) {
int n = d.a() - Base;
if( (int) stores.size() > n ) {
BasicRecStore *rs = stores[n];
if( rs )
if( rs ) {
assert( rs->fileNumber == n );
return *rs;
}
}
return *initStore(n);
}
BasicRecStore& store(const char *ns) {
char buf[256];
char *p = buf;
while( 1 ) {
if( *ns == '$' ) *p = '_';
else
*p = *ns;
if( *ns == 0 )
break;
p++; ns++;
}
assert( p - buf < (int) sizeof(buf) );
BasicRecStore *&rs = storesByNs[buf];
string nskey = mknskey(ns);
BasicRecStore *&rs = storesByNsKey[nskey];
if( rs )
return *rs;
initStoreByNs(buf);
initStoreByNs(ns, nskey);
return *rs;
}
@ -117,9 +114,9 @@ private:
void dump();
public:
/* all public functions (except constructor) use the mutex */
/* all public functions (except constructor) should use the mutex */
RecCache(unsigned sz) : recsize(sz) {
RecCache(unsigned recsz) : recsize(recsz) {
nnodes = 0;
newest = oldest = 0;
}