mongodb/buildscripts/mergerepositories.py

#!/usr/bin/python

from __future__ import with_statement
from libcloud.types import Provider
from libcloud.providers import get_driver
from libcloud.drivers.ec2 import EC2NodeDriver, NodeImage
from libcloud.base import Node, NodeImage, NodeSize, NodeState

# libcloud's SSH client seems to be one of those pointless wrappers
# that (at the moment) both doesn't add anything to the thing it wraps
# (Paramiko) and also fails to expose the underlying thing's features.
# What's wrong with people?
#from libcloud.ssh import SSHClient

import time
import sys
import settings
import subprocess
import os
import socket

EC2 = get_driver(Provider.EC2)
EC2Driver=EC2NodeDriver(settings.id, settings.key)

def tryEC2():

    image=NodeImage('ami-bf07ead6', 'ubuntu 10.4', EC2)
    size=NodeSize('m1.large', 'large', None, None, None, None, EC2)

    node = None
    try:
        node = EC2Driver.create_node(image=image, name="ubuntu-test", size=size, keyname="kp1", securitygroup=['default', 'dist-slave', 'buildbot-slave'])
        print node
        print node.id
        while node.state == NodeState.PENDING: 
            time.sleep(3)
    finally:
        if node:
            node.destroy()


class node(object):
    def initWait(self):
        while 1:
            n=None
            # EC2 sometimes takes a while to report a node.
            for i in range(6):
                nodes = [n for n in self.list_nodes() if (n.id==self.node.id)]
                if len(nodes)>0:
                    n=nodes[0]
                    break
                else:
                    time.sleep(10)
            if not n:
                raise Exception("couldn't find node with id %s" % self.node.id)
            if n.state == NodeState.PENDING: 
                time.sleep(10)
            else:
                self.node = n
                break
        print "ok"
        # Now wait for the node's sshd to be accepting connections.
        print "waiting for ssh"
        sshwait = True
        if sshwait == False:
            return
        while sshwait:
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            try:
                try:
                    s.connect((self.node.public_ip[0], 22))
                    sshwait = False
                    print "connected on port 22 (ssh)"
                    time.sleep(15) # arbitrary timeout, in case the
                    # remote sshd is slow.
                except socket.error, err:
                    pass
            finally:
                s.close()
                time.sleep(3) # arbitrary timeout
        print "ok"

    def __enter__(self):
        return self
        
    def __exit__(self, arg0, arg1, arg2):
        print "shutting down node %s" % self.node
        self.node.destroy()

# I don't think libcloud's Nodes implement __enter__ and __exit__, and
# I like the with statement for ensuring that we don't leak nodes when
# we don't have to.
class ec2node(node):
    def list_nodes(self):
        return EC2Driver.list_nodes()

class ubuntuNode(ec2node):
    def __init__(self):
        image=NodeImage('ami-bf07ead6', 'ubuntu 10.4', EC2)
        size=NodeSize('m1.large', 'large', None, None, None, None, EC2)

        self.node = EC2Driver.create_node(image=image, name="ubuntu-test", size=size, securitygroup=['default', 'dist-slave', 'buildbot-slave'], keyname='kp1')

class centosNode(ec2node):
    def __init__(self):
        image=NodeImage('ami-ccb35ea5', 'ubuntu 10.4', EC2)
        size=NodeSize('m1.large', 'large', None, None, None, None, EC2)

        self.node = EC2Driver.create_node(image=image, name="ubuntu-test", size=size, securitygroup=['default', 'dist-slave', 'buildbot-slave'], keyname='kp1')

class rackspaceNode(node):
    def list_nodes(self):
        self.conn.list_nodes()

class fedora11Node(rackspaceNode):
    def __init__(self):
        driver = get_driver(Provider.RACKSPACE)
        self.conn = driver(settings.rackspace_account, settings.rackspace_api_key)
        string='Fedora 11'
        images=filter(lambda x: (x.name.find(string) > -1), self.conn.list_images())
        sizes=self.conn.list_sizes()
        sizes.sort(cmp=lambda x,y: int(x.ram)<int(y.ram))
        node = None
        if len(images) != 1:
            raise "too many images with \"%s\" in the name" % string
        image = images[0]
        self.node = self.conn.create_node(image=image, name=string, size=sizes[0])
        print self.node
        self.password = self.node.extra['password']

class Err(Exception):
    pass

def merge_yum_repo(dir, outdir):
    dirtail=dir.rstrip('\/').split('/')[-1]
    keyfile=settings.makedist['ssh_keyfile']
    makeyumrepo="""find . -name RPMS | while read dir; do (cd $dir/.. && createrepo .); done"""
    with centosNode() as centos:
        centos.initWait()
        print centos.node
        run_for_effect(["scp", "-o", "StrictHostKeyChecking no","-i", keyfile, "-r", dir, "root@"+centos.node.public_ip[0]+":"])
        run_for_effect(["ssh", "-o", "StrictHostKeyChecking no","-i", keyfile, "root@"+centos.node.public_ip[0], "cd ./" + dirtail + " && " + makeyumrepo])
        run_for_effect(["scp", "-o", "StrictHostKeyChecking no", "-i", keyfile, "-r", "root@"+centos.node.public_ip[0]+":./"+dirtail +'/*', outdir])

        

def merge_apt_repo(dir, outdir):
    dirtail=dir.rstrip('\/').split('/')[-1]

    gpgdir=settings.makedist['gpg_homedir']
    keyfile=settings.makedist['ssh_keyfile']

    makeaptrepo="""for x in debian ubuntu; do (cd $x; for d in `find . -name *.deb | sed 's|^./||; s|/[^/]*$||'  | sort -u`; do dpkg-scanpackages $d > $d/Packages; gzip -9c $d/Packages > $d/Packages.gz; done) ; done"""
    makereleaseprologue="""Origin: 10gen
Label: 10gen
Suite: 10gen
Codename: VVVVVV
Version: VVVVVV
Architectures: i386 amd64
Components: 10gen
Description: 10gen packages"""
    makeaptrelease="""find . -maxdepth 3 -mindepth 3 | while read d; do ( cd $d && (echo '%s' | sed s/VVVVVV/$(basename $(pwd))/;  apt-ftparchive release .) > /tmp/Release && mv /tmp/Release .  && gpg -r `gpg  --list-keys | grep uid | awk '{print $(NF)}'` --no-secmem-warning --no-tty -abs --output Release.gpg Release ); done""" % makereleaseprologue
    with ubuntuNode() as ubuntu:
        ubuntu.initWait()
        print ubuntu.node
        run_for_effect(["ssh", "-o", "StrictHostKeyChecking no","-i", keyfile, "ubuntu@"+ubuntu.node.public_ip[0], "sudo", "sh", "-c", "\"export DEBIAN_FRONTEND=noninteractive; apt-get update; apt-get -y install debhelper\""])
        run_for_effect(["scp", "-o", "StrictHostKeyChecking no","-i", keyfile, "-r", dir, "ubuntu@"+ubuntu.node.public_ip[0]+":"])        
        run_for_effect(["scp", "-o", "StrictHostKeyChecking no","-i", keyfile, "-r", gpgdir, "ubuntu@"+ubuntu.node.public_ip[0]+":.gnupg"])
        run_for_effect(["ssh", "-o", "StrictHostKeyChecking no","-i", keyfile, "ubuntu@"+ubuntu.node.public_ip[0], "sh", "-c",  "\"ls -lR ./" + dirtail + "\""])
        run_for_effect(["ssh", "-o", "StrictHostKeyChecking no","-i", keyfile, "ubuntu@"+ubuntu.node.public_ip[0], "cd ./"+dirtail + " && " + makeaptrepo])
        run_for_effect(["ssh", "-o", "StrictHostKeyChecking no","-i", keyfile, "ubuntu@"+ubuntu.node.public_ip[0], "cd ./"+dirtail + " && " + makeaptrelease])
        run_for_effect(["scp", "-o", "StrictHostKeyChecking no", "-i", keyfile, "-r", "ubuntu@"+ubuntu.node.public_ip[0]+":./"+dirtail +'/*', outdir])


def run_for_effect(argv):
    print " ".join(argv)
    r=subprocess.Popen(argv).wait()
    if r!=0:
        raise Err("subprocess %s exited %d" % (argv, r))

if __name__ == "__main__":
    (flavor, dir, outdir) = sys.argv[-3:]

    if flavor == "deb":
        merge_apt_repo(dir, outdir)
    elif flavor == "rpm":
        merge_yum_repo(dir, outdir)
    else:
        Err("unknown pkg flavor %s" % flavor)
    # TODO: yum repositories

    
    #main()
    #tryRackSpace()