0
0
mirror of https://github.com/mongodb/mongo.git synced 2024-12-01 09:32:32 +01:00
mongodb/tools/tool.cpp

440 lines
14 KiB
C++
Raw Normal View History

/*
* Copyright (C) 2010 10gen Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
2009-01-27 21:16:09 +01:00
// Tool.cpp
2009-09-24 17:02:00 +02:00
#include "tool.h"
2009-01-27 21:16:09 +01:00
#include <iostream>
#include <boost/filesystem/operations.hpp>
#include <pcrecpp.h>
2009-01-27 21:16:09 +01:00
#include "util/file_allocator.h"
#include "util/password.h"
2009-01-27 21:16:09 +01:00
using namespace std;
using namespace mongo;
namespace po = boost::program_options;
2010-02-28 18:36:40 +01:00
namespace mongo {
CmdLine cmdLine;
Tool::Tool( string name , DBAccess access , string defaultDB ,
string defaultCollection , bool usesstdout ) :
_name( name ) , _db( defaultDB ) , _coll( defaultCollection ) ,
2010-06-09 19:20:35 +02:00
_usesstdout(usesstdout), _noconnection(false), _autoreconnect(false), _conn(0), _paired(false) {
2010-02-28 18:36:40 +01:00
_options = new po::options_description( "options" );
2010-02-21 16:42:38 +01:00
_options->add_options()
2010-02-28 18:36:40 +01:00
("help","produce help message")
("verbose,v", "be more verbose (include multiple times for more verbosity e.g. -vvvvv)")
;
2010-11-02 15:49:43 +01:00
if ( access & REMOTE_SERVER )
_options->add_options()
("host,h",po::value<string>(), "mongo host to connect to (\"left,right\" for pairs)" )
("port",po::value<string>(), "server port. Can also use --host hostname:port" )
2010-11-02 15:49:43 +01:00
("ipv6", "enable IPv6 support (disabled by default)")
("username,u",po::value<string>(), "username" )
("password,p", new PasswordValue( &_password ), "password" )
;
2010-11-02 15:49:43 +01:00
if ( access & LOCAL_SERVER )
2010-02-28 18:36:40 +01:00
_options->add_options()
2010-06-03 22:50:51 +02:00
("dbpath",po::value<string>(), "directly access mongod database "
"files in the given path, instead of connecting to a mongod "
"server - needs to lock the data directory, so cannot be "
"used if a mongod is currently accessing the same path" )
2010-02-28 18:36:40 +01:00
("directoryperdb", "if dbpath specified, each db is in a separate directory" )
2010-11-02 15:49:43 +01:00
;
if ( access & SPECIFY_DBCOL )
_options->add_options()
("db,d",po::value<string>(), "database to use" )
("collection,c",po::value<string>(), "collection to use (some commands)" )
2010-02-28 18:36:40 +01:00
;
_hidden_options = new po::options_description( name + " hidden options" );
/* support for -vv -vvvv etc. */
for (string s = "vv"; s.length() <= 10; s.append("v")) {
_hidden_options->add_options()(s.c_str(), "verbose");
}
}
2009-01-27 21:16:09 +01:00
2010-02-28 18:36:40 +01:00
Tool::~Tool(){
delete( _options );
delete( _hidden_options );
if ( _conn )
delete _conn;
}
2009-01-27 21:16:09 +01:00
2010-02-28 18:36:40 +01:00
void Tool::printHelp(ostream &out) {
printExtraHelp(out);
_options->print(out);
2010-07-12 19:17:34 +02:00
printExtraHelpAfter(out);
2010-02-28 18:36:40 +01:00
}
2010-02-28 18:36:40 +01:00
int Tool::main( int argc , char ** argv ){
static StaticObserver staticObserver;
2010-02-28 18:36:40 +01:00
cmdLine.prealloc = false;
2010-02-28 18:36:40 +01:00
boost::filesystem::path::default_name_check( boost::filesystem::no_check );
2009-06-25 22:11:38 +02:00
2010-02-28 18:36:40 +01:00
_name = argv[0];
2010-02-28 18:36:40 +01:00
/* using the same style as db.cpp */
int command_line_style = (((po::command_line_style::unix_style ^
po::command_line_style::allow_guessing) |
po::command_line_style::allow_long_disguise) ^
po::command_line_style::allow_sticky);
try {
po::options_description all_options("all options");
all_options.add(*_options).add(*_hidden_options);
2010-02-28 18:36:40 +01:00
po::store( po::command_line_parser( argc , argv ).
options(all_options).
positional( _positonalOptions ).
style(command_line_style).run() , _params );
2010-02-28 18:36:40 +01:00
po::notify( _params );
} catch (po::error &e) {
cerr << "ERROR: " << e.what() << endl << endl;
printHelp(cerr);
return EXIT_BADOPTIONS;
}
2009-01-27 21:16:09 +01:00
// hide password from ps output
for (int i=0; i < (argc-1); ++i){
if (!strcmp(argv[i], "-p") || !strcmp(argv[i], "--password")){
char* arg = argv[i+1];
while (*arg){
*arg++ = 'x';
}
}
}
2010-02-28 18:36:40 +01:00
if ( _params.count( "help" ) ){
printHelp(cout);
2010-02-28 18:36:40 +01:00
return 0;
}
2010-02-28 18:36:40 +01:00
if ( _params.count( "verbose" ) ) {
logLevel = 1;
}
2010-02-28 18:36:40 +01:00
for (string s = "vv"; s.length() <= 10; s.append("v")) {
if (_params.count(s)) {
logLevel = s.length();
}
}
2010-04-16 07:15:09 +02:00
preSetup();
2009-07-13 20:09:43 +02:00
2010-02-28 18:36:40 +01:00
bool useDirectClient = hasParam( "dbpath" );
2010-04-16 07:15:09 +02:00
2010-02-28 18:36:40 +01:00
if ( ! useDirectClient ) {
_host = "127.0.0.1";
if ( _params.count( "host" ) )
_host = _params["host"].as<string>();
if ( _params.count( "port" ) )
_host += ':' + _params["port"].as<string>();
2010-04-16 07:15:09 +02:00
if ( _noconnection ){
// do nothing
}
else {
2010-02-28 18:36:40 +01:00
string errmsg;
ConnectionString cs = ConnectionString::parse( _host , errmsg );
if ( ! cs.isValid() ){
cerr << "invalid hostname [" << _host << "] " << errmsg << endl;
2010-02-28 18:36:40 +01:00
return -1;
}
_conn = cs.connect( errmsg );
if ( ! _conn ){
cerr << "couldn't connect to [" << _host << "] " << errmsg << endl;
2010-02-28 18:36:40 +01:00
return -1;
}
(_usesstdout ? cout : cerr ) << "connected to: " << _host << endl;
}
}
else {
2010-02-28 18:36:40 +01:00
if ( _params.count( "directoryperdb" ) ) {
directoryperdb = true;
}
assert( lastError.get( true ) );
2010-02-28 18:36:40 +01:00
Client::initThread("tools");
_conn = new DBDirectClient();
_host = "DIRECT";
static string myDbpath = getParam( "dbpath" );
dbpath = myDbpath.c_str();
try {
acquirePathLock();
}
2010-07-02 18:03:07 +02:00
catch ( DBException& ){
cerr << endl << "If you are running a mongod on the same "
"path you should connect to that instead of direct data "
"file access" << endl << endl;
dbexit( EXIT_CLEAN );
return -1;
}
2010-02-28 18:36:40 +01:00
theFileAllocator().start();
}
2009-06-25 22:11:38 +02:00
2010-02-28 18:36:40 +01:00
if ( _params.count( "db" ) )
_db = _params["db"].as<string>();
2009-06-25 22:11:38 +02:00
2010-02-28 18:36:40 +01:00
if ( _params.count( "collection" ) )
_coll = _params["collection"].as<string>();
2009-09-01 21:43:23 +02:00
2010-02-28 18:36:40 +01:00
if ( _params.count( "username" ) )
_username = _params["username"].as<string>();
2009-08-12 22:31:22 +02:00
if ( _params.count( "password" )
&& ( _password.empty() ) ) {
_password = askPassword();
}
2009-09-01 21:43:23 +02:00
if (_params.count("ipv6"))
enableIPv6();
2010-02-28 18:36:40 +01:00
int ret = -1;
try {
ret = run();
}
catch ( DBException& e ){
cerr << "assertion: " << e.toString() << endl;
ret = -1;
}
2009-10-27 16:44:22 +01:00
2010-02-28 18:36:40 +01:00
if ( currentClient.get() )
currentClient->shutdown();
2010-02-02 03:35:54 +01:00
2010-02-28 18:36:40 +01:00
if ( useDirectClient )
dbexit( EXIT_CLEAN );
return ret;
}
2009-08-12 22:31:22 +02:00
2010-02-28 18:36:40 +01:00
DBClientBase& Tool::conn( bool slaveIfPaired ){
if ( slaveIfPaired && _conn->type() == ConnectionString::SET )
return ((DBClientReplicaSet*)_conn)->slaveConn();
2010-02-28 18:36:40 +01:00
return *_conn;
}
bool Tool::isMaster() {
if ( hasParam("dbpath") ) {
return true;
}
BSONObj info;
bool isMaster;
bool ok = conn().isMaster(isMaster, &info);
if (ok && !isMaster) {
cerr << "ERROR: trying to write to non-master " << conn().toString() << endl;
cerr << "isMaster info: " << info << endl;
return false;
}
return true;
}
2010-02-28 18:36:40 +01:00
void Tool::addFieldOptions(){
add_options()
2010-05-08 17:46:40 +02:00
("fields,f" , po::value<string>() , "comma separated list of field names e.g. -f name,age" )
2010-02-28 18:36:40 +01:00
("fieldFile" , po::value<string>() , "file with fields names - 1 per line" )
;
}
2009-10-12 21:05:42 +02:00
2010-02-28 18:36:40 +01:00
void Tool::needFields(){
2009-10-12 21:05:42 +02:00
2010-02-28 18:36:40 +01:00
if ( hasParam( "fields" ) ){
BSONObjBuilder b;
2009-10-12 21:05:42 +02:00
2010-02-28 18:36:40 +01:00
string fields_arg = getParam("fields");
pcrecpp::StringPiece input(fields_arg);
2009-10-12 21:05:42 +02:00
2010-02-28 18:36:40 +01:00
string f;
pcrecpp::RE re("([#\\w\\.\\s\\-]+),?" );
2010-02-28 18:36:40 +01:00
while ( re.Consume( &input, &f ) ){
_fields.push_back( f );
2010-07-20 18:58:23 +02:00
b.append( f , 1 );
2010-02-28 18:36:40 +01:00
}
2009-10-12 21:05:42 +02:00
2010-02-28 18:36:40 +01:00
_fieldsObj = b.obj();
return;
}
2009-10-12 21:05:42 +02:00
2010-02-28 18:36:40 +01:00
if ( hasParam( "fieldFile" ) ){
string fn = getParam( "fieldFile" );
if ( ! exists( fn ) )
throw UserException( 9999 , ((string)"file: " + fn ) + " doesn't exist" );
const int BUF_SIZE = 1024;
char line[ 1024 + 128];
ifstream file( fn.c_str() );
BSONObjBuilder b;
while ( file.rdstate() == ios_base::goodbit ){
file.getline( line , BUF_SIZE );
const char * cur = line;
while ( isspace( cur[0] ) ) cur++;
2010-06-30 00:08:07 +02:00
if ( cur[0] == '\0' )
2010-02-28 18:36:40 +01:00
continue;
_fields.push_back( cur );
b.append( cur , 1 );
}
_fieldsObj = b.obj();
return;
2009-10-12 21:05:42 +02:00
}
2010-02-28 18:36:40 +01:00
throw UserException( 9998 , "you need to specify fields" );
2009-10-12 21:05:42 +02:00
}
2010-02-28 18:36:40 +01:00
void Tool::auth( string dbname ){
if ( ! dbname.size() )
dbname = _db;
2010-02-28 18:36:40 +01:00
if ( ! ( _username.size() || _password.size() ) )
return;
2009-08-12 22:31:22 +02:00
2010-02-28 18:36:40 +01:00
string errmsg;
if ( _conn->auth( dbname , _username , _password , errmsg ) )
return;
2009-08-12 22:31:22 +02:00
2010-02-28 18:36:40 +01:00
// try against the admin db
string err2;
if ( _conn->auth( "admin" , _username , _password , errmsg ) )
return;
2009-09-01 21:43:23 +02:00
2010-02-28 18:36:40 +01:00
throw UserException( 9997 , (string)"auth failed: " + errmsg );
}
2009-09-01 21:43:23 +02:00
BSONTool::BSONTool( const char * name, DBAccess access , bool objcheck )
: Tool( name , access , "" , "" ) , _objcheck( objcheck ){
2010-06-09 17:31:30 +02:00
add_options()
("objcheck" , "validate object before inserting" )
("filter" , po::value<string>() , "filter to apply before inserting" )
;
}
int BSONTool::run(){
_objcheck = hasParam( "objcheck" );
if ( hasParam( "filter" ) )
_matcher.reset( new Matcher( fromjson( getParam( "filter" ) ) ) );
return doRun();
}
long long BSONTool::processFile( const path& root ){
string fileString = root.string();
long long fileLength = file_size( root );
if ( fileLength == 0 ) {
out() << "file " << fileString << " empty, skipping" << endl;
return 0;
}
2010-07-08 02:01:34 +02:00
FILE* file = fopen( fileString.c_str() , "rb" );
if ( ! file ){
2010-06-09 17:31:30 +02:00
log() << "error opening file: " << fileString << endl;
return 0;
}
2010-07-08 02:01:34 +02:00
#if !defined(__sunos__) && defined(POSIX_FADV_SEQUENTIAL)
posix_fadvise(fileno(file), 0, fileLength, POSIX_FADV_SEQUENTIAL);
#endif
2010-06-09 17:31:30 +02:00
log(1) << "\t file size: " << fileLength << endl;
long long read = 0;
long long num = 0;
long long processed = 0;
const int BUF_SIZE = 1024 * 1024 * 5;
boost::scoped_array<char> buf_holder(new char[BUF_SIZE]);
char * buf = buf_holder.get();
ProgressMeter m( fileLength );
while ( read < fileLength ) {
2010-07-19 15:40:15 +02:00
int readlen = fread(buf, 4, 1, file);
2010-06-09 17:31:30 +02:00
int size = ((int*)buf)[0];
if ( size >= BUF_SIZE ){
cerr << "got an object of size: " << size << " terminating..." << endl;
}
uassert( 10264 , "invalid object size" , size < BUF_SIZE );
2010-07-19 15:40:15 +02:00
readlen = fread(buf+4, size-4, 1, file);
2010-06-09 17:31:30 +02:00
BSONObj o( buf );
if ( _objcheck && ! o.valid() ){
cerr << "INVALID OBJECT - going try and pring out " << endl;
cerr << "size: " << size << endl;
BSONObjIterator i(o);
while ( i.more() ){
BSONElement e = i.next();
try {
e.validate();
}
catch ( ... ){
cerr << "\t\t NEXT ONE IS INVALID" << endl;
}
cerr << "\t name : " << e.fieldName() << " " << e.type() << endl;
cerr << "\t " << e << endl;
}
}
if ( _matcher.get() == 0 || _matcher->matches( o ) ){
gotObject( o );
processed++;
}
read += o.objsize();
num++;
m.hit( o.objsize() );
}
uassert( 10265 , "counts don't match" , m.done() == fileLength );
out() << "\t " << m.hits() << " objects found" << endl;
if ( _matcher.get() )
out() << "\t " << processed << " objects processed" << endl;
return processed;
}
void setupSignals( bool inFork ){}
2009-08-12 22:31:22 +02:00
}