#!/usr/bin/env python # makedist.py: make a distro package (on an EC2 (or sometimes # RackSpace) instance) # For ease of use, put a file called settings.py someplace in your # sys.path, containing something like the following: # makedist = { # # This gets supplied to EC2 to rig up an ssh key for # # the remote user. # "ec2_sshkey" : "key-id", # # And so we need to tell our ssh processes where to find the # # appropriate public key file. # "ssh_keyfile" : "/path/to/key-id-file" # } # Notes: although there is a Python library for accessing EC2 as a web # service, it seemed as if it would be less work to just shell out to # the three EC2 management tools we use. # To make a distribution we must: # 1. Fire up an EC2 AMI suitable for building. # 2. Get any build-dependencies and configurations onto the remote host. # 3. Fetch the mongodb source. # 4. Run the package building tools. # 5. Save the package archives someplace permanent (eventually we # ought to install them into a public repository for the distro). # Unimplemented: # 6. Fire up an EC2 AMI suitable for testing whether the packages # install. # 7. Check whether the packages install and run. # The implementations of steps 1, 2, 4, 5, 6, and 7 will depend on the # distro of host we're talking to (Ubuntu, CentOS, Debian, etc.). from __future__ import with_statement import subprocess import sys import signal import getopt import socket import time import os.path import tempfile import string import settings from libcloud.types import Provider from libcloud.providers import get_driver from libcloud.drivers.ec2 import EC2NodeDriver, NodeImage from libcloud.base import Node, NodeImage, NodeSize, NodeState from libcloud.ssh import ParamikoSSHClient # For the moment, we don't handle any of the errors we raise, so it # suffices to have a simple subclass of Exception that just # stringifies according to a desired format. class SimpleError(Exception): def __init__(self, *args): self.args = args def __str__(self): return self.args[0] % self.args[1:] class SubcommandError(SimpleError): def __init__(self, *args): self.status = args[2] super(SubcommandError, self).__init__(*args) class BaseConfigurator (object): def __init__ (self, **kwargs): self.configuration = [] self.arch=kwargs["arch"] self.distro_name=kwargs["distro_name"] self.distro_version=kwargs["distro_version"] def lookup(self, what, dist, vers, arch): for (wht, seq) in self.configuration: if what == wht: for ((dpat, vpat, apat), payload) in seq: # For the moment, our pattern facility is just "*" or exact match. if ((dist == dpat or dpat == "*") and (vers == vpat or vpat == "*") and (arch == apat or apat == "*")): return payload if getattr(self, what, False): return getattr(self, what) else: raise SimpleError("couldn't find a%s %s configuration for dist=%s, version=%s, arch=%s", "n" if ("aeiouAEIOU".find(what[0]) > -1) else "", what, dist, vers, arch) def default(self, what): return self.lookup(what, self.distro_name, self.distro_version, self.arch) def findOrDefault(self, dict, what): return (dict[what] if what in dict else self.lookup(what, self.distro_name, self.distro_version, self.arch)) class BaseHostConfigurator (BaseConfigurator): def __init__(self, **kwargs): super(BaseHostConfigurator, self).__init__(**kwargs) self.configuration += [("distro_arch", ((("debian", "*", "x86_64"), "amd64"), (("ubuntu", "*", "x86_64"), "amd64"), (("debian", "*", "x86"), "i386"), (("ubuntu", "*", "x86"), "i386"), (("centos", "*", "x86_64"), "x86_64"), (("fedora", "*", "x86_64"), "x86_64"), (("centos", "*", "x86"), "i386"), (("fedora", "*", "x86"), "i386"), (("*", "*", "x86_64"), "x86_64"), (("*", "*", "x86"), "x86"))) , ] class LocalHost(object): @classmethod def runLocally(cls, argv): print "running %s" % argv r = subprocess.Popen(argv).wait() if r != 0: raise SubcommandError("subcommand %s exited %d", argv, r) class EC2InstanceConfigurator(BaseConfigurator): def __init__(self, **kwargs): super(EC2InstanceConfigurator, self).__init__(**kwargs) self.configuration += [("ec2_ami", ((("ubuntu", "10.4", "x86_64"), "ami-bf07ead6"), (("ubuntu", "10.4", "x86"), "ami-f707ea9e"), (("ubuntu", "9.10", "x86_64"), "ami-55739e3c"), (("ubuntu", "9.10", "x86"), "ami-bb709dd2"), (("ubuntu", "9.4", "x86_64"), "ami-eef61587"), (("ubuntu", "9.4", "x86"), "ami-ccf615a5"), (("ubuntu", "8.10", "x86"), "ami-c0f615a9"), (("ubuntu", "8.10", "x86_64"), "ami-e2f6158b"), (("ubuntu", "8.4", "x86"), "ami59b35f30"), (("ubuntu", "8.4", "x86_64"), "ami-27b35f4e"), (("debian", "5.0", "x86"), "ami-dcf615b5"), (("debian", "5.0", "x86_64"), "ami-f0f61599"), (("centos", "5.4", "x86"), "ami-f8b35e91"), (("centos", "5.4", "x86_64"), "ami-ccb35ea5"), (("fedora", "8", "x86_64"), "ami-2547a34c"), (("fedora", "8", "x86"), "ami-5647a33f"))), ("rackspace_imgname", ((("fedora", "11", "x86_64"), "Fedora 11"), (("fedora", "12", "x86_64"), "Fedora 12"))), ("ec2_mtype", ((("*", "*", "x86"), "m1.small"), (("*", "*", "x86_64"), "m1.large"))), ] class nodeWrapper(object): def __init__(self, configurator, **kwargs): self.terminate = False if "no_terminate" in kwargs else True def getHostname(self): return self.node.public_ip[0] # FIXME private_ip? def initwait(self): print "waiting for node to spin up" # Wait for EC2 to tell us the node is running. while 1: n=None # EC2 sometimes takes a while to report a node. for i in range(6): nodes = [n for n in self.list_nodes() if (n.id==self.node.id)] if len(nodes)>0: n=nodes[0] break else: time.sleep(10) if not n: raise Exception("couldn't find node with id %s" % self.node.id) if n.state == NodeState.PENDING: time.sleep(10) else: self.node = n break print "ok" # Now wait for the node's sshd to be accepting connections. print "waiting for ssh" sshwait = True if sshwait == False: return while sshwait: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: try: s.connect((self.node.public_ip[0], 22)) sshwait = False print "connected on port 22 (ssh)" time.sleep(15) # arbitrary timeout, in case the # remote sshd is slow. except socket.error, err: pass finally: s.close() time.sleep(3) # arbitrary timeout print "ok" def __enter__(self): self.start() # Note: we don't do an initwait() in __enter__ because if an # exception is raised during __enter__, __exit__ doesn't get # run (and by inspection RackSpace doesn't let you kill a node # that hasn't finished booting yet). return self def __exit__(self, type, value, traceback): self.stop() def stop(self): if self.terminate: print "Destroying node %s" % self.node.id self.node.destroy() else: print "Not terminating EC2 instance %s." % self.node.id def setup(self): pass class EC2Instance (nodeWrapper): def __init__(self, configurator, **kwargs): super(EC2Instance, self).__init__(configurator, **kwargs) # Stuff we need to start an instance: AMI name, key and cert # files. AMI and mtype default to configuration in this file, # but can be overridden. self.ec2_ami = configurator.findOrDefault(kwargs, "ec2_ami") self.ec2_mtype = configurator.findOrDefault(kwargs, "ec2_mtype") self.use_internal_name = True if "use_internal_name" in kwargs else False self.ec2_sshkey=kwargs["ec2_sshkey"] # FIXME: this needs to be a commandline option self.ec2_groups = ["default", "buildbot-slave", "dist-slave"] def start(self): "Fire up a fresh EC2 instance." EC2 = get_driver(Provider.EC2) self.driver = EC2NodeDriver(settings.id, settings.key) image = NodeImage(self.ec2_ami, self.ec2_ami, EC2) size = NodeSize(self.ec2_mtype, self.ec2_mtype, None, None, None, None, EC2) self.node = self.driver.create_node(image=image, name=self.ec2_ami, size=size, keyname=self.ec2_sshkey, securitygroup=self.ec2_groups) print "Created node %s" % self.node.id def list_nodes(self): return self.driver.list_nodes() class SshConnectionConfigurator (BaseConfigurator): def __init__(self, **kwargs): super(SshConnectionConfigurator, self).__init__(**kwargs) self.configuration += [("ssh_login", # FLAW: this actually depends more on the AMI # than the triple. ((("debian", "*", "*"), "root"), (("ubuntu", "10.4", "*"), "ubuntu"), (("ubuntu", "9.10", "*"), "ubuntu"), (("ubuntu", "9.4", "*"), "root"), (("ubuntu", "8.10", "*"), "root"), (("ubuntu", "8.4", "*"), "ubuntu"), (("fedora", "*", "*"), "root"), (("centos", "*", "*"), "root"))), ] class SshConnection (object): def __init__(self, configurator, **kwargs): # Stuff we need to talk to the thing properly self.ssh_login = configurator.findOrDefault(kwargs, "ssh_login") self.ssh_host = kwargs["ssh_host"] self.ssh_keyfile=kwargs["ssh_keyfile"] # Gets set to False when we think we can ssh in. self.sshwait = True def initSsh(self): ctlpath="/tmp/ec2-ssh-%s-%s-%s" % (self.ssh_host, self.ssh_login, os.getpid()) argv = ["ssh", "-o", "StrictHostKeyChecking no", "-M", "-o", "ControlPath %s" % ctlpath, "-v", "-l", self.ssh_login, "-i", self.ssh_keyfile, self.ssh_host] print "Setting up ssh master connection with %s" % argv self.sshproc = subprocess.Popen(argv) self.ctlpath = ctlpath def __enter__(self): self.initSsh() return self def __exit__(self, type, value, traceback): os.kill(self.sshproc.pid, signal.SIGTERM) self.sshproc.wait() def runRemotely(self, argv): """Run a command on the host.""" LocalHost.runLocally(["ssh", "-o", "StrictHostKeyChecking no", "-S", self.ctlpath, "-l", self.ssh_login, "-i", self.ssh_keyfile, self.ssh_host] + argv) def sendFiles(self, files): for (localfile, remotefile) in files: LocalHost.runLocally(["scp", "-o", "StrictHostKeyChecking no", "-o", "ControlMaster auto", "-o", "ControlPath %s" % self.ctlpath, "-i", self.ssh_keyfile, "-rv", localfile, self.ssh_login + "@" + self.ssh_host + ":" + ("" if remotefile is None else remotefile) ]) def recvFiles(self, files): for (remotefile, localfile) in files: LocalHost.runLocally(["scp", "-o", "StrictHostKeyChecking no", "-o", "ControlMaster auto", "-o", "ControlPath %s" % self.ctlpath, "-i", self.ssh_keyfile, "-rv", self.ssh_login + "@" + self.ssh_host + ":" + remotefile, "." if localfile is None else localfile ]) class ScriptFileConfigurator (BaseConfigurator): deb_productdir = "dists" rpm_productdir = "/usr/src/redhat/RPMS" # FIXME: this could be # ~/redhat/RPMS or # something elsewhere preamble_commands = """ set -x # verbose execution, for debugging set -e # errexit, stop on errors """ # Strictly speaking, we don't need to mangle debian files on rpm # systems (and vice versa), but (a) it doesn't hurt anything to do # so, and (b) mangling files the same way everywhere could # conceivably help uncover bugs in the hideous hideous sed # programs we're running here. (N.B., for POSIX wonks: POSIX sed # doesn't support either in-place file editing, which we use # below. So if we end up wanting to run these mangling commands # e.g., on a BSD, we'll need to make them fancier.) mangle_files_commands =""" # On debianoids, the package names in the changelog and control file # must agree, and only files in a subdirectory of debian/ matching the # package name will get included in the .deb, so we also have to mangle # the rules file. ( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && sed -i '1s/.*([^)]*)/{pkg_name}{pkg_name_suffix} ({pkg_version})/' debian/changelog ) || exit 1 ( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && sed -i 's/^Source:.*/Source: {pkg_name}{pkg_name_suffix}/; s/^Package:.*mongodb/Package: {pkg_name}{pkg_name_suffix}\\ Conflicts: {pkg_name_conflicts}/' debian/control; ) || exit 1 ( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && sed -i 's|$(CURDIR)/debian/mongodb/|$(CURDIR)/debian/{pkg_name}{pkg_name_suffix}/|g' debian/rules) || exit 1 ( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && sed -i 's|debian/mongodb.manpages|debian/{pkg_name}{pkg_name_suffix}.manpages|g' debian/rules) || exit 1 ( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && sed -i '/^Name:/s/.*/Name: {pkg_name}{pkg_name_suffix}/; /^Version:/s/.*/Version: {pkg_version}/; /Requires.*mongo/s/mongo/{pkg_name}{pkg_name_suffix}/;' rpm/mongo.spec ) # Debian systems require some ridiculous workarounds to get an init # script at /etc/init.d/mongodb when the packge name isn't the init # script name. Note: dh_installinit --name won't work, because that # option would require the init script under debian/ to be named # mongodb. ( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && ln debian/init.d debian/{pkg_name}{pkg_name_suffix}.mongodb.init && ln debian/mongodb.upstart debian/{pkg_name}{pkg_name_suffix}.mongodb.upstart && sed -i 's/dh_installinit/dh_installinit --name=mongodb/' debian/rules) || exit 1 ( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && cat debian/rules) ( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && cat rpm/mongo.spec) """ # If we're just packaging up nightlies, do this: nightly_build_mangle_files=""" ( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && sed -i '/scons[[:space:]]*$/d; s^scons.*install^mkdir -p debian/{pkg_name}{pkg_name_suffix} \&\& wget http://downloads.mongodb.org/linux/mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz \&\& tar xzvf mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz \&\& find `tar tzf mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz | sed "s|/.*||" | sort -u | head -n1` -mindepth 1 -maxdepth 1 -type d | xargs -n1 -IARG mv -v ARG debian/{pkg_name}{pkg_name_suffix}/usr \&\& (rm debian/{pkg_name}{pkg_name_suffix}/usr/bin/mongosniff || true)^' debian/rules) ( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && sed -i 's/^BuildRequires:.*//; s/scons.*\ -c//; s/scons.*\ all//; s^scons.*install^(mkdir -p $RPM_BUILD_ROOT/usr ; cd /tmp \&\& curl http://downloads.mongodb.org/linux/mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz > mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz \&\& tar xzvf mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz \&\& find `tar tzf mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz | sed "s|/.*||" | sort -u | head -n1` -mindepth 1 -maxdepth 1 -type d | xargs -n1 -IARG cp -pRv ARG $RPM_BUILD_ROOT/usr \&\& (rm -r $RPM_BUILD_ROOT/usr/bin/mongosniff $RPM_BUILD_ROOT/usr/lib64/libmongoclient.a $RPM_BUILD_ROOT/usr/lib/libmongoclient.a $RPM_BUILD_ROOT/usr/include/mongo || true))^' rpm/mongo.spec) # Upstream nightlies no longer contain libmongoclient. ( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && sed -i '/%package devel/{{N;N;d;}}; /%description devel/{{N;N;N;N;N;d;}}; /%files devel/{{N;N;N;d;}};' rpm/mongo.spec ) ( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && cat debian/rules) ( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && cat rpm/mongo.spec) """ #$RPM_BUILD_ROOT/usr/lib/libmongoclient.a $RPM_BUILD_ROOT/usr/lib64/libmongoclient.a mangle_files_for_new_deb_xulrunner_commands = """ ( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && sed -i 's/xulrunner-dev/xulrunner-1.9.2-dev/g' debian/control ) """ mangle_files_for_ancient_redhat_commands = """ # Ancient RedHats ship with very old boosts and non-UTF8-aware js # libraries, so we need to link statically to those. ( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && sed -i 's|^scons.*((inst)all)|scons --prefix=$RPM_BUILD_ROOT/usr --extralib=nspr4 --staticlib=boost_system-mt,boost_thread-mt,boost_filesystem-mt,boost_program_options-mt,js $1|' rpm/mongo.spec ) """ deb_prereq_commands = """ # Configure debconf to never prompt us for input. export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install -y {pkg_prereq_str} """ deb_build_commands=""" mkdir -p "{pkg_product_dir}/{distro_version}/10gen/binary-{distro_arch}" mkdir -p "{pkg_product_dir}/{distro_version}/10gen/source" ( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}"; debuild ) || exit 1 # Try installing it dpkg -i {pkg_name}{pkg_name_suffix}*.deb ps ax | grep mongo || {{ echo "no running mongo" >/dev/stderr; exit 1; }} dpkg --remove $(for f in {pkg_name}{pkg_name_suffix}*.deb ; do echo ${{f%%_*}}; done) dpkg --purge $(for f in {pkg_name}{pkg_name_suffix}*.deb ; do echo ${{f%%_*}}; done) cp {pkg_name}{pkg_name_suffix}*.deb "{pkg_product_dir}/{distro_version}/10gen/binary-{distro_arch}" cp {pkg_name}{pkg_name_suffix}*.dsc "{pkg_product_dir}/{distro_version}/10gen/source" cp {pkg_name}{pkg_name_suffix}*.tar.gz "{pkg_product_dir}/{distro_version}/10gen/source" dpkg-scanpackages "{pkg_product_dir}/{distro_version}/10gen/binary-{distro_arch}" /dev/null | gzip -9c > "{pkg_product_dir}/{distro_version}/10gen/binary-{distro_arch}/Packages.gz" dpkg-scansources "{pkg_product_dir}/{distro_version}/10gen/source" /dev/null | gzip -9c > "{pkg_product_dir}/{distro_version}/10gen/source/Sources.gz" """ rpm_prereq_commands = """ rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/{distro_arch}/epel-release-5-3.noarch.rpm yum -y install {pkg_prereq_str} """ rpm_build_commands=""" for d in BUILD BUILDROOT RPMS SOURCES SPECS SRPMS; do mkdir -p {rpmbuild_dir}/$d; done cp -v "{pkg_name}{pkg_name_suffix}-{pkg_version}/rpm/mongo.spec" {rpmbuild_dir}/SPECS/{pkg_name}{pkg_name_suffix}.spec tar -cpzf {rpmbuild_dir}/SOURCES/"{pkg_name}{pkg_name_suffix}-{pkg_version}".tar.gz "{pkg_name}{pkg_name_suffix}-{pkg_version}" rpmbuild -ba --target={distro_arch} {rpmbuild_dir}/SPECS/{pkg_name}{pkg_name_suffix}.spec # FIXME: should install the rpms, check if mongod is running. """ # FIXME: this is clean, but adds 40 minutes or so to the build process. old_rpm_precommands = """ yum install -y bzip2-devel python-devel libicu-devel chrpath zlib-devel nspr-devel readline-devel ncurses-devel # FIXME: this is just some random URL found on rpmfind some day in 01/2010. wget ftp://194.199.20.114/linux/EPEL/5Client/SRPMS/js-1.70-8.el5.src.rpm rpm -ivh js-1.70-8.el5.src.rpm sed -i 's/XCFLAGS.*$/XCFLAGS=\"%{{optflags}} -fPIC -DJS_C_STRINGS_ARE_UTF8\" \\\\/' /usr/src/redhat/SPECS/js.spec rpmbuild -ba /usr/src/redhat/SPECS/js.spec rpm -Uvh /usr/src/redhat/RPMS/{distro_arch}/js-1.70-8.{distro_arch}.rpm rpm -Uvh /usr/src/redhat/RPMS/{distro_arch}/js-devel-1.70-8.{distro_arch}.rpm # FIXME: this is just some random URL found on rpmfind some day in 01/2010. wget ftp://195.220.108.108/linux/sourceforge/g/project/gr/gridiron2/support-files/FC10%20source%20RPMs/boost-1.38.0-1.fc10.src.rpm rpm -ivh boost-1.38.0-1.fc10.src.rpm rpmbuild -ba /usr/src/redhat/SPECS/boost.spec rpm -ivh /usr/src/redhat/RPMS/{distro_arch}/boost-1.38.0-1.{distro_arch}.rpm rpm -ivh /usr/src/redhat/RPMS/{distro_arch}/boost-devel-1.38.0-1.{distro_arch}.rpm """ # This horribleness is an attempt to work around ways that you're # not really meant to package things for Debian unless you are # Debian. # On very old Debianoids, libboost--dev will be some old # boost that's not as thready as we want, but which Eliot says # will work; on very new Debianoids, libbost--dev is what we # want. unversioned_deb_boost_prereqs = ["libboost-thread-dev", "libboost-filesystem-dev", "libboost-program-options-dev", "libboost-date-time-dev", "libboost-dev"] # On some in-between Debianoids, libboost--dev is still a # 1.34, but 1.35 packages are available, so we want those. versioned_deb_boost_prereqs = ["libboost-thread1.35-dev", "libboost-filesystem1.35-dev", "libboost-program-options1.35-dev", "libboost-date-time1.35-dev", "libboost1.35-dev"] unversioned_deb_xulrunner_prereqs = ["xulrunner-dev"] old_versioned_deb_xulrunner_prereqs = ["xulrunner-1.9-dev"] new_versioned_deb_xulrunner_prereqs = ["xulrunner-1.9.2-dev"] common_deb_prereqs = [ "build-essential", "dpkg-dev", "libreadline-dev", "libpcap-dev", "libpcre3-dev", "git-core", "scons", "debhelper", "devscripts", "git-core" ] centos_preqres = ["js-devel", "readline-devel", "pcre-devel", "gcc-c++", "scons", "rpm-build", "git" ] fedora_prereqs = ["js-devel", "readline-devel", "pcre-devel", "gcc-c++", "scons", "rpm-build", "git", "curl" ] def __init__(self, **kwargs): super(ScriptFileConfigurator, self).__init__(**kwargs) # FIXME: this method is disabled until we get back around to # actually building from source. if None: # kwargs["mongo_version"][0] == 'r': self.get_mongo_commands = """ wget -Otarball.tgz "http://github.com/mongodb/mongo/tarball/{mongo_version}"; tar xzf tarball.tgz mv "`tar tzf tarball.tgz | sed 's|/.*||' | sort -u | head -n1`" "{pkg_name}{pkg_name_suffix}-{pkg_version}" """ else: self.get_mongo_commands = """ git clone git://github.com/mongodb/mongo.git """ # This is disabled for the moment. it's for building the # tip of some versioned branch. if None: #kwargs['mongo_version'][0] == 'v': self.get_mongo_commands +=""" ( cd mongo && git archive --prefix="{pkg_name}{pkg_name_suffix}-{pkg_version}/" "`git log origin/{mongo_version} | sed -n '1s/^commit //p;q'`" ) | tar xf - """ else: self.get_mongo_commands += """ ( cd mongo && git archive --prefix="{pkg_name}{pkg_name_suffix}-{pkg_version}/" "{mongo_version}" ) | tar xf - """ if "local_mongo_dir" in kwargs: self.mangle_files_commands = """( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && rm -rf debian rpm && cp -pvR ~/pkg/* . ) """ + self.mangle_files_commands self.configuration += [("pkg_product_dir", ((("ubuntu", "*", "*"), self.deb_productdir), (("debian", "*", "*"), self.deb_productdir), (("fedora", "*", "*"), "~/rpmbuild/RPMS"), (("centos", "*", "*"), "/usr/src/redhat/RPMS"))), ("pkg_prereqs", ((("ubuntu", "9.4", "*"), self.versioned_deb_boost_prereqs + self.unversioned_deb_xulrunner_prereqs + self.common_deb_prereqs), (("ubuntu", "9.10", "*"), self.unversioned_deb_boost_prereqs + self.unversioned_deb_xulrunner_prereqs + self.common_deb_prereqs), (("ubuntu", "10.4", "*"), self.unversioned_deb_boost_prereqs + self.new_versioned_deb_xulrunner_prereqs + self.common_deb_prereqs), (("ubuntu", "8.10", "*"), self.versioned_deb_boost_prereqs + self.unversioned_deb_xulrunner_prereqs + self.common_deb_prereqs), (("ubuntu", "8.4", "*"), self.unversioned_deb_boost_prereqs + self.old_versioned_deb_xulrunner_prereqs + self.common_deb_prereqs), (("debian", "5.0", "*"), self.versioned_deb_boost_prereqs + self.unversioned_deb_xulrunner_prereqs + self.common_deb_prereqs), (("fedora", "*", "*"), self.fedora_prereqs), (("centos", "5.4", "*"), self.centos_preqres))), # FIXME: this is deprecated ("commands", ((("debian", "*", "*"), self.deb_prereq_commands + self.get_mongo_commands + self.mangle_files_commands + self.deb_build_commands), (("ubuntu", "10.4", "*"), self.preamble_commands + self.deb_prereq_commands + self.get_mongo_commands + self.mangle_files_commands + self.mangle_files_for_new_deb_xulrunner_commands + self.deb_build_commands), (("ubuntu", "*", "*"), self.preamble_commands + self.deb_prereq_commands + self.get_mongo_commands + self.mangle_files_commands + self.deb_build_commands), (("centos", "*", "*"), self.preamble_commands + self.old_rpm_precommands + self.rpm_prereq_commands + self.get_mongo_commands + self.mangle_files_commands + self.mangle_files_for_ancient_redhat_commands + self.rpm_build_commands), (("fedora", "*", "*"), self.preamble_commands + self.old_rpm_precommands + self.rpm_prereq_commands + self.get_mongo_commands + self.mangle_files_commands + self.rpm_build_commands))), ("preamble_commands", ((("*", "*", "*"), self.preamble_commands), )), ("install_prereqs", ((("debian", "*", "*"), self.deb_prereq_commands), (("ubuntu", "*", "*"), self.deb_prereq_commands), (("centos", "*", "*"), self.rpm_prereq_commands), (("fedora", "*", "*"), self.rpm_prereq_commands))), ("get_mongo", ((("*", "*", "*"), self.get_mongo_commands), )), ("mangle_mongo", ((("debian", "*", "*"), self.mangle_files_commands), (("ubuntu", "10.4", "*"), self.mangle_files_commands + self.mangle_files_for_new_deb_xulrunner_commands), (("ubuntu", "*", "*"), self.mangle_files_commands), (("centos", "*", "*"), self.mangle_files_commands + self.mangle_files_for_ancient_redhat_commands), (("fedora", "*", "*"), self.mangle_files_commands))), ("build_prerequisites", ((("fedora", "*", "*"), self.old_rpm_precommands), (("centos", "*", "*"), self.old_rpm_precommands), (("*", "*", "*"), ''))), ("install_for_packaging", ((("debian", "*", "*"),""), (("ubuntu", "*", "*"),""), (("fedora", "*", "*"), ""), (("centos", "*", "*"),""))), ("build_package", ((("debian", "*", "*"), self.deb_build_commands), (("ubuntu", "*", "*"), self.deb_build_commands), (("fedora", "*", "*"), self.rpm_build_commands), (("centos", "*", "*"), self.rpm_build_commands))), ("pkg_name", ((("debian", "*", "*"), "mongodb"), (("ubuntu", "*", "*"), "mongodb"), (("centos", "*", "*"), "mongo"), (("fedora", "*", "*"), "mongo"))), # FIXME: there should be a command-line argument for this. ("pkg_name_conflicts", ((("*", "*", "*"), ["", "-stable", "-unstable", "-snapshot", "-oldstable"]), )), ("rpmbuild_dir", ((("fedora", "*", "*"), "~/rpmbuild"), (("centos", "*", "*"), "/usr/src/redhat"), (("*", "*","*"), ''), )), ] class ScriptFile(object): def __init__(self, configurator, **kwargs): self.configurator = configurator self.mongo_version_spec = kwargs['mongo_version_spec'] self.mongo_arch = kwargs["arch"] if kwargs["arch"] == "x86_64" else "i686" self.pkg_prereqs = configurator.default("pkg_prereqs") self.pkg_name = configurator.default("pkg_name") self.pkg_product_dir = configurator.default("pkg_product_dir") #self.formatter = configurator.default("commands") self.distro_name = configurator.default("distro_name") self.distro_version = configurator.default("distro_version") self.distro_arch = configurator.default("distro_arch") def bogoformat(self, fmt, **kwargs): r = '' i = 0 while True: c = fmt[i] if c in '{}': i+=1 c2=fmt[i] if c2 == c: r+=c else: j=i while True: p=fmt[j:].find('}') if p == -1: raise Exception("malformed format string starting at %d: no closing brace" % i) else: j+=p if len(fmt) > (j+1) and fmt[j+1]=='}': j+=2 else: break key = fmt[i:j] r+=kwargs[key] i=j else: r+=c i+=1 if i==len(fmt): return r def fmt(self, formatter, **kwargs): try: return string.Formatter.format(formatter, kwargs) finally: return self.bogoformat(formatter, **kwargs) def genscript(self): script='' formatter = self.configurator.default("preamble_commands") + self.configurator.default("install_prereqs") script+=self.fmt(formatter, distro_name=self.distro_name, distro_version=self.distro_version, distro_arch=self.distro_arch, pkg_name=self.pkg_name, pkg_product_dir=self.pkg_product_dir, mongo_arch=self.mongo_arch, pkg_prereq_str=" ".join(self.pkg_prereqs), ) specs=self.mongo_version_spec.split(',') for spec in specs: (version, pkg_name_suffix, pkg_version) = parse_mongo_version_spec(spec) mongo_version = version if version[0] != 'n' else ('HEAD' if version == 'nlatest' else 'r'+version[1:]) #'HEAD' mongo_pub_version = version.lstrip('n') if version[0] in 'n' else 'latest' pkg_name_suffix = pkg_name_suffix if pkg_name_suffix else '' pkg_version = pkg_version pkg_name_conflicts = self.configurator.default("pkg_name_conflicts") if pkg_name_suffix else [] pkg_name_conflicts.remove(pkg_name_suffix) if pkg_name_suffix and pkg_name_suffix in pkg_name_conflicts else [] formatter = self.configurator.default("get_mongo") + self.configurator.default("mangle_mongo") + (self.configurator.nightly_build_mangle_files if version[0] == 'n' else '') +(self.configurator.default("build_prerequisites") if version[0] != 'n' else '') + self.configurator.default("install_for_packaging") + self.configurator.default("build_package") script+=self.fmt(formatter, mongo_version=mongo_version, distro_name=self.distro_name, distro_version=self.distro_version, distro_arch=self.distro_arch, pkg_prereq_str=" ".join(self.pkg_prereqs), pkg_name=self.pkg_name, pkg_name_suffix=pkg_name_suffix, pkg_version=pkg_version, pkg_product_dir=self.pkg_product_dir, # KLUDGE: rpm specs and deb # control files use # comma-separated conflicts, # but there's no reason to # suppose this works elsewhere pkg_name_conflicts = ", ".join([self.pkg_name+conflict for conflict in pkg_name_conflicts]), mongo_arch=self.mongo_arch, mongo_pub_version=mongo_pub_version, rpmbuild_dir=self.configurator.default('rpmbuild_dir')) script+='rm -rf mongo' return script def __enter__(self): self.localscript=None # One of tempfile or I is very stupid. (fh, name) = tempfile.mkstemp('', "makedist.", ".") try: pass finally: os.close(fh) with open(name, 'w+') as fh: fh.write(self.genscript()) self.localscript=name return self def __exit__(self, type, value, traceback): if self.localscript: os.unlink(self.localscript) class Configurator(SshConnectionConfigurator, EC2InstanceConfigurator, ScriptFileConfigurator, BaseHostConfigurator): def __init__(self, **kwargs): super(Configurator, self).__init__(**kwargs) class rackspaceInstance(nodeWrapper): def __init__(self, configurator, **kwargs): super(rackspaceInstance, self).__init__(configurator, **kwargs) self.imgname=configurator.default('rackspace_imgname') def start(self): driver = get_driver(Provider.RACKSPACE) self.conn = driver(settings.rackspace_account, settings.rackspace_api_key) name=self.imgname+'-'+str(os.getpid()) images=filter(lambda x: (x.name.find(self.imgname) > -1), self.conn.list_images()) sizes=self.conn.list_sizes() sizes.sort(cmp=lambda x,y: int(x.ram) 1: raise Exception("too many images with \"%s\" in the name" % self.imgname) if len(images) < 1: raise Exception("too few images with \"%s\" in the name" % self.imgname) image = images[0] self.node = self.conn.create_node(image=image, name=name, size=sizes[0]) # Note: the password is available only in the response to the # create_node request, not in subsequent list_nodes() # requests; so although the node objects we get back from # list_nodes() are usuable for most things, we must hold onto # the initial password. self.password = self.node.extra['password'] print self.node def list_nodes(self): return self.conn.list_nodes() def setup(self): self.putSshKey() def putSshKey(self): keyfile=settings.makedist['ssh_keyfile'] ssh = ParamikoSSHClient(hostname = self.node.public_ip[0], password = self.password) ssh.connect() print "putting ssh public key" ssh.put(".ssh/authorized_keys", contents=open(keyfile+'.pub').read(), chmod=0600) print "ok" def parse_mongo_version_spec (spec): foo = spec.split(":") mongo_version = foo[0] # this can be a commit id, a # release id "r1.2.2", or a branch name # starting with v. if len(foo) > 1: pkg_name_suffix = foo[1] if len(foo) > 2 and foo[2]: pkg_version = foo[2] else: pkg_version = time.strftime("%Y%m%d") if not pkg_name_suffix: if mongo_version[0] in ["r", "v"]: nums = mongo_version.split(".") if int(nums[1]) % 2 == 0: pkg_name_suffix = "-stable" else: pkg_name_suffix = "-unstable" else: pkg_name_suffix = "" return (mongo_version, pkg_name_suffix, pkg_version) def main(): # checkEnvironment() (kwargs, args) = processArguments() (rootdir, distro_name, distro_version, arch, mongo_version_spec) = args[:5] # FIXME: there are a few other characters that we can't use in # file names on Windows, in case this program really needs to run # there. distro_name = distro_name.replace('/', '-').replace('\\', '-') distro_version = distro_version.replace('/', '-').replace('\\', '-') arch = arch.replace('/', '-').replace('\\', '-') try: import settings if "makedist" in dir ( settings ): for key in ["ec2_sshkey", "ssh_keyfile", "gpg_homedir" ]: if key not in kwargs and key in settings.makedist: kwargs[key] = settings.makedist[key] except Exception, err: print "No settings: %s. Continuing anyway..." % err pass kwargs["distro_name"] = distro_name kwargs["distro_version"] = distro_version kwargs["arch"] = arch kwargs['mongo_version_spec'] = mongo_version_spec kwargs["localdir"] = rootdir # FIXME: this should also include the mongo version or something. # if "subdirs" in kwargs: # kwargs["localdir"] = "%s/%s/%s/%s/%s" % (rootdir, distro_name, distro_version, arch, kwargs["mongo_version"]) # else: kwargs['gpg_homedir'] = kwargs["gpg_homedir"] if "gpg_homedir" in kwargs else os.path.expanduser("~/.gnupg") configurator = Configurator(**kwargs) LocalHost.runLocally(["mkdir", "-p", kwargs["localdir"]]) with ScriptFile(configurator, **kwargs) as script: with open(script.localscript) as f: print """# Going to run the following on a fresh AMI:""" print f.read() time.sleep(10) # FIXME: it's not the best to have two different pathways for # the different hosting services, but... with EC2Instance(configurator, **kwargs) if kwargs['distro_name'] != 'fedora' else rackspaceInstance(configurator, **kwargs) as host: host.initwait() host.setup() kwargs["ssh_host"] = host.getHostname() with SshConnection(configurator, **kwargs) as ssh: ssh.runRemotely(["uname -a; ls /"]) ssh.runRemotely(["mkdir", "pkg"]) if "local_mongo_dir" in kwargs: ssh.sendFiles([(kwargs["local_mongo_dir"]+'/'+d, "pkg") for d in ["rpm", "debian"]]) ssh.sendFiles([(kwargs['gpg_homedir'], ".gnupg")]) ssh.sendFiles([(script.localscript, "makedist.sh")]) ssh.runRemotely((["sudo"] if ssh.ssh_login != "root" else [])+ ["sh", "makedist.sh"]) ssh.recvFiles([(script.pkg_product_dir, kwargs['localdir'])]) def processArguments(): # flagspec [ (short, long, argument?, description, argname)* ] flagspec = [ ("?", "usage", False, "Print a (useless) usage message", None), ("h", "help", False, "Print a help message and exit", None), ("N", "no-terminate", False, "Leave the EC2 instance running at the end of the job", None), ("S", "subdirs", False, "Create subdirectories of the output directory based on distro name, version, and architecture", None), ("I", "use-internal-name", False, "Use the EC2 internal hostname for sshing", None), (None, "gpg-homedir", True, "Local directory of gpg junk", "STRING"), (None, "local-mongo-dir", True, "Copy packaging files from local mongo checkout", "DIRECTORY"), ] shortopts = "".join([t[0] + (":" if t[2] else "") for t in flagspec if t[0] is not None]) longopts = [t[1] + ("=" if t[2] else "") for t in flagspec] try: opts, args = getopt.getopt(sys.argv[1:], shortopts, longopts) except getopt.GetoptError, err: print str(err) sys.exit(2) # Normalize the getopt-parsed options. kwargs = {} for (opt, arg) in opts: flag = opt opt = opt.lstrip("-") if flag[:2] == '--': #long opt kwargs[opt.replace('-', '_')] = arg elif flag[:1] == "-": #short opt ok = False for tuple in flagspec: if tuple[0] == opt: ok = True kwargs[tuple[1].replace('-', '_')] = arg break if not ok: raise SimpleError("this shouldn't happen: unrecognized option flag: %s", opt) else: raise SimpleError("this shouldn't happen: non-option returned from getopt()") if "help" in kwargs: print "Usage: %s [OPTIONS] DIRECTORY DISTRO DISTRO-VERSION ARCHITECTURE MONGO-VERSION-SPEC" % sys.argv[0] print """Build some packages on new EC2 AMI instances, leave packages under DIRECTORY. MONGO-VERSION-SPEC has the syntax Commit(:Pkg-Name-Suffix(:Pkg-Version)). If Commit starts with an 'r', build from a tagged release; if Commit starts with an 'n', package up a nightly build; if Commit starts with a 'v', build from the HEAD of a version branch; otherwise, build whatever git commit is identified by Commit. Pkg-Name-Suffix gets appended to the package name, and defaults to "-stable" and "-unstable" if Commit looks like it designates a stable or unstable release/branch, respectively. Pkg-Version is used as the package version, and defaults to YYYYMMDD. Examples: HEAD # build a snapshot of HEAD, name the package # "mongodb", use YYYYMMDD for the version HEAD:-snap # build a snapshot of HEAD, name the package # "mongodb-snap", use YYYYMMDD for the version HEAD:-snap:123 # build a snapshot of HEAD, name the package # "mongodb-snap", use 123 for the version HEAD:-suffix:1.3 # build a snapshot of HEAD, name the package # "mongodb-snapshot", use "1.3 for the version r1.2.3 # build a package of the 1.2.3 release, call it "mongodb-stable", # make the package version YYYYMMDD. v1.2:-stable: # build a package of the HEAD of the 1.2 branch decafbad:-foo:123 # build git commit "decafbad", call the package # "mongodb-foo" with package version 123. Options:""" for t in flagspec: print "%-20s\t%s." % ("%4s--%s%s:" % ("-%s, " % t[0] if t[0] else "", t[1], ("="+t[4]) if t[4] else ""), t[3]) print """ Mandatory arguments to long options are also mandatory for short options.""" sys.exit(0) if "usage" in kwargs: print "Usage: %s [OPTIONS] OUTPUT-DIR DISTRO-NAME DISTRO-VERSION ARCHITECTURE MONGO-VERSION-SPEC" % sys.argv[0] sys.exit(0) return (kwargs, args) if __name__ == "__main__": main() # Examples: # ./makedist.py /tmp/ubuntu ubuntu 8.10 x86_64 HEAD:-snapshot,v1.4:-stable,v1.5:-unstable # ./makedist.py /tmp/ubuntu ubuntu 8.10 x86_64 nlatest:-snapshot,n1.4.2:-stable,n1.5.0:-unstable