0
0
mirror of https://github.com/mongodb/mongo.git synced 2024-12-01 09:32:32 +01:00

Significant rejiggering in packaging scripts.

This commit is contained in:
Richard Kreuter 2010-04-23 16:31:58 -04:00
parent fe092fc624
commit 5100a7c813
3 changed files with 135 additions and 93 deletions

View File

@ -31,8 +31,8 @@ def pushrepo(repodir):
# equivalent to a/b.
s3name1='distros-archive/'+time.strftime('%Y%m%d')+tail
s3name2='distros'+tail
# s3cp(bucket, fn, s3name1)
# s3cp(bucket, fn, s3name2)
s3cp(bucket, fn, s3name1)
s3cp(bucket, fn, s3name2)
if s3name.endswith('.deb'):
newdebs.append(s3name)
# FIXME: we ought to clean out old debs eventually, but this will
@ -108,19 +108,18 @@ def parse_mongo_version_spec(spec):
l+=['']
return l
def logfh(distro, distro_version, arch, mongo_version):
prefix = "%s-%s-%s-%s.log." % (distro, distro_version, arch, mongo_version)
def logfh(distro, distro_version, arch):
prefix = "%s-%s-%s.log." % (distro, distro_version, arch)
# This is a NamedTemporaryFile mostly so that I can tail(1) them
# as we go.
return tempfile.NamedTemporaryFile("w+b", -1, prefix=prefix)
def spawn(distro, distro_version, arch, spec, directory, opts):
(mongo_version, suffix, pkg_version) = parse_mongo_version_spec(spec)
argv = ["python", "makedist.py"] + opts + [ directory, distro, distro_version, arch ] + [ spec ]
# cmd = "mkdir -p %s; cd %s; touch foo.deb; echo %s %s %s %s %s | tee Packages " % ( directory, directory, directory, distro, distro_version, arch, mongo_version )
# print cmd
# argv = ["sh", "-c", cmd]
fh = logfh(distro, distro_version, arch, mongo_version)
fh = logfh(distro, distro_version, arch)
print >> fh, "Running %s" % argv
# it's often handy to be able to run these things at the shell
# manually. FIXME: this ought to be slightly less than thoroughly
@ -153,9 +152,9 @@ def wait(procs, winfh, losefh, winners, losers):
next
if pid:
[tup] = [tup for tup in procs if tup[0].pid == pid]
(proc, logfh, distro, distro_version, arch, mongo_version) = tup
(proc, logfh, distro, distro_version, arch, spec) = tup
procs.remove(tup)
name = "%s %s %s %s" % (distro, distro_version, arch, mongo_version)
name = "%s %s %s" % (distro, distro_version, arch)
if os.WIFEXITED(stat):
if os.WEXITSTATUS(stat) == 0:
win(name, logfh, winfh)
@ -177,10 +176,9 @@ def __main__():
# Output from makedist.py goes here.
outputroot=tempfile.mkdtemp()
mergedir=tempfile.mkdtemp()
repodir=tempfile.mkdtemp()
print "makedist output under: %s\nmerge directory: %s\ncombined repo: %s\n" % (outputroot, mergedir, repodir)
print "makedist output under: %s\ncombined repo: %s\n" % (outputroot, repodir)
sys.stdout.flush()
# Add more dist/version/architecture tuples as they're supported.
dists = (("ubuntu", "10.4"),
@ -190,7 +188,7 @@ def __main__():
("debian", "5.0"),
("centos", "5.4"),)
arches = ("x86", "x86_64")
mongos = branches.split(',')
# mongos = branches.split(',')
# Run a makedist for each distro/version/architecture tuple above.
winners = []
losers = []
@ -198,18 +196,13 @@ def __main__():
losefh=tempfile.TemporaryFile()
procs = []
count = 0
for ((distro, distro_version), arch, spec) in gen([dists, arches, mongos]):
for ((distro, distro_version), arch, spec) in gen([dists, arches, [branches]]):
count+=1
opts = makedistopts
(mongo_version,_,_) = parse_mongo_version_spec(spec)
# blech: the "Packages.gz" metadata files in a Debian
# repository will clobber each other unless we make a
# different "repository" for each mongo version we're
# building.
if distro in ["debian", "ubuntu"]:
outputdir = "%s/%s/%s" % (outputroot, mongo_version, distro)
outputdir = "%s/deb/%s" % (outputroot, distro)
elif distro in ["centos", "fedora", "redhat"]:
outputdir = "%s/%s/%s/%s/os" % (outputroot, mongo_version, distro, distro_version)
outputdir = "%s/rpm/%s/%s/os" % (outputroot, distro, distro_version)
else:
raise Exception("unsupported distro %s" % distro)
#opts += ["--subdirs"]
@ -240,13 +233,12 @@ def __main__():
sys.stdout.flush()
sys.stderr.flush()
merge_directories_concatenating_conflicts(mergedir, glob.glob(outputroot+'/*'))
# this is sort of ridiculous, but the outputs from rpmbuild look
# like RPM/<arch>, but the repo wants to look like
# <arch>/RPM.
for dist in os.listdir(mergedir):
for dist in os.listdir(outputroot+'/rpm'):
if dist in ["centos", "fedora", "redhat"]:
distdir="%s/%s" % (mergedir, dist)
distdir="%s/rpm/%s" % (outputroot, dist)
rpmdirs = subprocess.Popen(["find", distdir, "-type", "d", "-a", "-name", "RPMS"], stdout=subprocess.PIPE).communicate()[0].split('\n')[:-1]
for rpmdir in rpmdirs:
for arch in os.listdir(rpmdir):
@ -255,16 +247,17 @@ def __main__():
os.rename("%s/%s" % (rpmdir, arch), "%s/RPMS" % (archdir,))
os.rmdir(rpmdir)
argv=["python", "mergerepositories.py", mergedir, repodir]
print "running %s" % argv
print " ".join(argv)
r = subprocess.Popen(argv).wait()
if r != 0:
raise Exception("mergerepositories.py exited %d" % r)
print repodir
for flavor in os.listdir(outputroot):
argv=["python", "mergerepositories.py", flavor, "%s/%s" % (outputroot, flavor), repodir]
print "running %s" % argv
print " ".join(argv)
r = subprocess.Popen(argv).wait()
if r != 0:
raise Exception("mergerepositories.py exited %d" % r)
print repodir
pushrepo(repodir)
shutil.rmtree(outputroot)
shutil.rmtree(mergedir)
shutil.rmtree(repodir)
return 0

View File

@ -363,7 +363,6 @@ class SshConnection (object):
def recvFiles(self, files):
self.sshWait()
print files
for (remotefile, localfile) in files:
LocalHost.runLocally(["scp", "-o", "StrictHostKeyChecking no",
"-o", "ControlMaster auto",
@ -448,8 +447,10 @@ mkdir -p "{pkg_product_dir}/{distro_version}/10gen/binary-{distro_arch}"
mkdir -p "{pkg_product_dir}/{distro_version}/10gen/source"
( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}"; debuild ) || exit 1
# Try installing it
dpkg -i *.deb
dpkg -i {pkg_name}{pkg_name_suffix}*.deb
ps ax | grep mongo || {{ echo "no running mongo" >/dev/stderr; exit 1; }}
dpkg --remove $(for f in {pkg_name}{pkg_name_suffix}*.deb ; do echo ${{f%%_*}}; done)
dpkg --purge $(for f in {pkg_name}{pkg_name_suffix}*.deb ; do echo ${{f%%_*}}; done)
cp {pkg_name}{pkg_name_suffix}*.deb "{pkg_product_dir}/{distro_version}/10gen/binary-{distro_arch}"
cp {pkg_name}{pkg_name_suffix}*.dsc "{pkg_product_dir}/{distro_version}/10gen/source"
cp {pkg_name}{pkg_name_suffix}*.tar.gz "{pkg_product_dir}/{distro_version}/10gen/source"
@ -522,7 +523,9 @@ mv "`tar tzf tarball.tgz | sed 's|/.*||' | sort -u | head -n1`" "{pkg_name}{pkg_
self.get_mongo_commands = """
git clone git://github.com/mongodb/mongo.git
"""
if kwargs['mongo_version'][0] == 'v':
# This is disabled for the moment. it's for building the
# tip of some versioned branch.
if None: #kwargs['mongo_version'][0] == 'v':
self.get_mongo_commands +="""
( cd mongo && git archive --prefix="{pkg_name}{pkg_name_suffix}-{pkg_version}/" "`git log origin/{mongo_version} | sed -n '1s/^commit //p;q'`" ) | tar xf -
"""
@ -623,18 +626,13 @@ git clone git://github.com/mongodb/mongo.git
class ScriptFile(object):
def __init__(self, configurator, **kwargs):
self.mongo_version = kwargs["mongo_version"] if kwargs['mongo_version'][0] != 'n' else 'HEAD'
self.mongo_pub_version = kwargs["mongo_version"].lstrip('n') if kwargs['mongo_version'][0] in 'n' else 'latest'
self.configurator = configurator
self.mongo_version_spec = kwargs['mongo_version_spec']
self.mongo_arch = kwargs["arch"] if kwargs["arch"] == "x86_64" else "i686"
self.pkg_version = kwargs["pkg_version"]
self.pkg_name_suffix = kwargs["pkg_name_suffix"] if "pkg_name_suffix" in kwargs else ""
self.pkg_prereqs = configurator.default("pkg_prereqs")
self.pkg_name = configurator.default("pkg_name")
self.pkg_product_dir = configurator.default("pkg_product_dir")
self.pkg_name_conflicts = configurator.default("pkg_name_conflicts") if self.pkg_name_suffix else []
self.pkg_name_conflicts.remove(self.pkg_name_suffix) if self.pkg_name_suffix and self.pkg_name_suffix in self.pkg_name_conflicts else []
#self.formatter = configurator.default("commands")
self.formatter = configurator.default("preamble_commands") + configurator.default("install_prereqs") + configurator.default("get_mongo") + configurator.default("mangle_mongo") + (configurator.nightly_build_mangle_files if kwargs['mongo_version'][0] == 'n' else '') +(configurator.default("build_prerequisites") if kwargs['mongo_version'][0] != 'n' else '') + configurator.default("install_for_packaging") + configurator.default("build_package")
self.distro_name = configurator.default("distro_name")
self.distro_version = configurator.default("distro_version")
self.distro_arch = configurator.default("distro_arch")
@ -677,25 +675,49 @@ class ScriptFile(object):
return self.bogoformat(formatter, **kwargs)
def genscript(self):
return self.fmt(self.formatter,
mongo_version=self.mongo_version,
distro_name=self.distro_name,
distro_version=self.distro_version,
distro_arch=self.distro_arch,
pkg_prereq_str=" ".join(self.pkg_prereqs),
pkg_name=self.pkg_name,
pkg_name_suffix=self.pkg_name_suffix,
pkg_version=self.pkg_version,
pkg_product_dir=self.pkg_product_dir,
# KLUDGE: rpm specs and deb
# control files use
# comma-separated conflicts,
# but there's no reason to
# suppose this works elsewhere
pkg_name_conflicts = ", ".join([self.pkg_name+conflict for conflict in self.pkg_name_conflicts]),
mongo_arch=self.mongo_arch,
mongo_pub_version=self.mongo_pub_version
)
script=''
formatter = self.configurator.default("preamble_commands") + self.configurator.default("install_prereqs")
script+=self.fmt(formatter,
distro_name=self.distro_name,
distro_version=self.distro_version,
distro_arch=self.distro_arch,
pkg_name=self.pkg_name,
pkg_product_dir=self.pkg_product_dir,
mongo_arch=self.mongo_arch,
pkg_prereq_str=" ".join(self.pkg_prereqs),
)
specs=self.mongo_version_spec.split(',')
for spec in specs:
(version, pkg_name_suffix, pkg_version) = parse_mongo_version_spec(spec)
mongo_version = version if version[0] != 'n' else 'HEAD'
mongo_pub_version = version.lstrip('n') if version[0] in 'n' else 'latest'
pkg_name_suffix = pkg_name_suffix if pkg_name_suffix else ''
pkg_version = pkg_version
pkg_name_conflicts = self.configurator.default("pkg_name_conflicts") if pkg_name_suffix else []
pkg_name_conflicts.remove(pkg_name_suffix) if pkg_name_suffix and pkg_name_suffix in pkg_name_conflicts else []
formatter = self.configurator.default("get_mongo") + self.configurator.default("mangle_mongo") + (self.configurator.nightly_build_mangle_files if version[0] == 'n' else '') +(self.configurator.default("build_prerequisites") if version[0] != 'n' else '') + self.configurator.default("install_for_packaging") + self.configurator.default("build_package")
script+=self.fmt(formatter,
mongo_version=mongo_version,
distro_name=self.distro_name,
distro_version=self.distro_version,
distro_arch=self.distro_arch,
pkg_prereq_str=" ".join(self.pkg_prereqs),
pkg_name=self.pkg_name,
pkg_name_suffix=pkg_name_suffix,
pkg_version=pkg_version,
pkg_product_dir=self.pkg_product_dir,
# KLUDGE: rpm specs and deb
# control files use
# comma-separated conflicts,
# but there's no reason to
# suppose this works elsewhere
pkg_name_conflicts = ", ".join([self.pkg_name+conflict for conflict in pkg_name_conflicts]),
mongo_arch=self.mongo_arch,
mongo_pub_version=mongo_pub_version
)
script+='rm -rf mongo'
return script
def __enter__(self):
self.localscript=None
@ -718,6 +740,28 @@ class Configurator(SshConnectionConfigurator, EC2InstanceConfigurator, ScriptFil
def __init__(self, **kwargs):
super(Configurator, self).__init__(**kwargs)
def parse_mongo_version_spec (spec):
foo = spec.split(":")
mongo_version = foo[0] # this can be a commit id, a
# release id "r1.2.2", or a branch name
# starting with v.
if len(foo) > 1:
pkg_name_suffix = foo[1]
if len(foo) > 2 and foo[2]:
pkg_version = foo[2]
else:
pkg_version = time.strftime("%Y%m%d")
if not pkg_name_suffix:
if mongo_version[0] in ["r", "v"]:
nums = mongo_version.split(".")
if int(nums[1]) % 2 == 0:
pkg_name_suffix = "-stable"
else:
pkg_name_suffix = "-unstable"
else:
pkg_name_suffix = ""
return (mongo_version, pkg_name_suffix, pkg_version)
def main():
# checkEnvironment()
@ -755,33 +799,15 @@ def main():
kwargs["distro_name"] = distro_name
kwargs["distro_version"] = distro_version
kwargs["arch"] = arch
foo = mongo_version_spec.split(":")
kwargs["mongo_version"] = foo[0] # this can be a commit id, a
# release id "r1.2.2", or a
# branch name starting with v.
if len(foo) > 1:
kwargs["pkg_name_suffix"] = foo[1]
if len(foo) > 2 and foo[2]:
kwargs["pkg_version"] = foo[2]
else:
kwargs["pkg_version"] = time.strftime("%Y%m%d")
kwargs['mongo_version_spec'] = mongo_version_spec
kwargs["localdir"] = rootdir
# FIXME: this should also include the mongo version or something.
if "subdirs" in kwargs:
kwargs["localdir"] = "%s/%s/%s/%s/%s" % (rootdir, distro_name, distro_version, arch, kwargs["mongo_version"])
else:
kwargs["localdir"] = rootdir
# if "subdirs" in kwargs:
# kwargs["localdir"] = "%s/%s/%s/%s/%s" % (rootdir, distro_name, distro_version, arch, kwargs["mongo_version"])
# else:
if "pkg_name_suffix" not in kwargs:
if kwargs["mongo_version"][0] in ["r", "v"]:
nums = kwargs["mongo_version"].split(".")
if int(nums[1]) % 2 == 0:
kwargs["pkg_name_suffix"] = "-stable"
else:
kwargs["pkg_name_suffix"] = "-unstable"
else:
kwargs["pkg_name_suffix"] = ""
kwargs['gpg_homedir'] = kwargs["gpg_homedir"] if "gpg_homedir" in kwargs else os.path.expanduser("~/.gnupg")

View File

@ -42,13 +42,7 @@ def tryEC2():
# I don't think libcloud's Nodes implement __enter__ and __exit__, and
# I like the with statement for ensuring that we don't leak nodes when
# we don't have to.
class ubuntuNode(object):
def __init__(self):
image=NodeImage('ami-bf07ead6', 'ubuntu 10.4', EC2)
size=NodeSize('m1.large', 'large', None, None, None, None, EC2)
self.node = EC2Driver.create_node(image=image, name="ubuntu-test", size=size, securitygroup=['default', 'dist-slave', 'buildbot-slave'], keyname='kp1')
class ec2node(object):
def initWait(self):
print "waiting for node to spin up"
# Wait for EC2 to tell us the node is running.
@ -99,6 +93,21 @@ class ubuntuNode(object):
print "shutting down node %s" % self.node
self.node.destroy()
class ubuntuNode(ec2node):
def __init__(self):
image=NodeImage('ami-bf07ead6', 'ubuntu 10.4', EC2)
size=NodeSize('m1.large', 'large', None, None, None, None, EC2)
self.node = EC2Driver.create_node(image=image, name="ubuntu-test", size=size, securitygroup=['default', 'dist-slave', 'buildbot-slave'], keyname='kp1')
class centosNode(ec2node):
def __init__(self):
image=NodeImage('ami-ccb35ea5', 'ubuntu 10.4', EC2)
size=NodeSize('m1.large', 'large', None, None, None, None, EC2)
self.node = EC2Driver.create_node(image=image, name="ubuntu-test", size=size, securitygroup=['default', 'dist-slave', 'buildbot-slave'], keyname='kp1')
def tryRackSpace():
driver=get_driver(Provider.RACKSPACE)
@ -124,7 +133,22 @@ def tryRackSpace():
class Err(Exception):
pass
def merge_yum_repo(dir, outdir):
dirtail=dir.rstrip('\/').split('/')[-1]
keyfile=settings.makedist['ssh_keyfile']
makeyumrepo="""find . -name RPMS | while read dir; do (cd $dir/.. && createrepo .); done"""
with centosNode() as centos:
centos.initWait()
print centos.node
run_for_effect(["scp", "-o", "StrictHostKeyChecking no","-i", keyfile, "-r", dir, "root@"+centos.node.public_ip[0]+":"])
run_for_effect(["ssh", "-o", "StrictHostKeyChecking no","-i", keyfile, "root@"+centos.node.public_ip[0], "cd ./" + dirtail + " && " + makeyumrepo])
run_for_effect(["scp", "-o", "StrictHostKeyChecking no", "-i", keyfile, "-r", "root@"+centos.node.public_ip[0]+":./"+dirtail +'/*', outdir])
def merge_apt_repo(dir, outdir):
dirtail=dir.rstrip('\/').split('/')[-1]
gpgdir=settings.makedist['gpg_homedir']
keyfile=settings.makedist['ssh_keyfile']
@ -142,7 +166,7 @@ Description: 10gen packages"""
ubuntu.initWait()
print ubuntu.node
run_for_effect(["ssh", "-o", "StrictHostKeyChecking no","-i", keyfile, "ubuntu@"+ubuntu.node.public_ip[0], "sudo", "sh", "-c", "\"export DEBIAN_FRONTEND=noninteractive; apt-get update; apt-get -y install debhelper\""])
run_for_effect(["scp", "-o", "StrictHostKeyChecking no","-i", keyfile, "-r", dir, "ubuntu@"+ubuntu.node.public_ip[0]+":"])
run_for_effect(["scp", "-o", "StrictHostKeyChecking no","-i", keyfile, "-r", dir, "ubuntu@"+ubuntu.node.public_ip[0]+":"])
run_for_effect(["scp", "-o", "StrictHostKeyChecking no","-i", keyfile, "-r", gpgdir, "ubuntu@"+ubuntu.node.public_ip[0]+":.gnupg"])
run_for_effect(["ssh", "-o", "StrictHostKeyChecking no","-i", keyfile, "ubuntu@"+ubuntu.node.public_ip[0], "sh", "-c", "\"ls -lR ./" + dirtail + "\""])
run_for_effect(["ssh", "-o", "StrictHostKeyChecking no","-i", keyfile, "ubuntu@"+ubuntu.node.public_ip[0], "cd ./"+dirtail + " && " + makeaptrepo])
@ -158,7 +182,6 @@ def run_for_effect(argv):
if __name__ == "__main__":
(flavor, dir, outdir) = sys.argv[-3:]
dirtail=dir.rstrip('\/').split('/')[-1]
if flavor == "deb":
merge_apt_repo(dir, outdir)