mirror of
https://github.com/mongodb/mongo.git
synced 2024-12-01 09:32:32 +01:00
359 lines
14 KiB
Python
359 lines
14 KiB
Python
#!/usr/bin/env python
|
|
|
|
import re
|
|
import sys
|
|
import os
|
|
import tempfile
|
|
import subprocess
|
|
import json
|
|
import urlparse
|
|
import tarfile
|
|
import signal
|
|
import threading
|
|
import traceback
|
|
import shutil
|
|
import errno
|
|
from contextlib import closing
|
|
# To ensure it exists on the system
|
|
import zipfile
|
|
|
|
#
|
|
# Useful script for installing multiple versions of MongoDB on a machine
|
|
# Only really tested/works on Linux.
|
|
#
|
|
|
|
|
|
def dump_stacks(signal, frame):
|
|
print "======================================"
|
|
print "DUMPING STACKS due to SIGUSR1 signal"
|
|
print "======================================"
|
|
threads = threading.enumerate()
|
|
|
|
print "Total Threads: " + str(len(threads))
|
|
|
|
for id, stack in sys._current_frames().items():
|
|
print "Thread %d" % (id)
|
|
print "".join(traceback.format_stack(stack))
|
|
print "======================================"
|
|
|
|
|
|
def get_version_parts(version, for_sorting=False):
|
|
"""Returns a list containing the components of the version string
|
|
as numeric values. This function can be used for numeric sorting
|
|
of version strings such as '2.6.0-rc1' and '2.4.0' when the
|
|
'for_sorting' parameter is specified as true."""
|
|
|
|
RC_OFFSET = -100
|
|
version_parts = re.split(r'\.|-', version)
|
|
|
|
if version_parts[-1] == "pre":
|
|
# Prior to improvements for how the version string is managed within the server
|
|
# (SERVER-17782), the binary archives would contain a trailing "-pre".
|
|
version_parts.pop()
|
|
|
|
if version_parts[-1].startswith("rc"):
|
|
# RC versions are weighted down to allow future RCs and general
|
|
# releases to be sorted in ascending order (e.g., 2.6.0-rc1,
|
|
# 2.6.0-rc2, 2.6.0).
|
|
version_parts[-1] = int(version_parts[-1][2:]) + RC_OFFSET
|
|
elif version_parts[0].startswith("v") and version_parts[-1] == "latest":
|
|
version_parts[0] = version_parts[0][1:]
|
|
# The "<branchname>-latest" versions are weighted the highest when a particular major
|
|
# release is requested.
|
|
version_parts[-1] = float("inf")
|
|
elif for_sorting:
|
|
# We want to have the number of components in the resulting version parts match the number
|
|
# of components in the 'version' string if we aren't going to be using them for sorting.
|
|
# Otherwise, we append an additional 0 to non-RC releases so that version lists like
|
|
# [2, 6, 0, -100] and [2, 6, 0, 0] sort in ascending order.
|
|
version_parts.append(0)
|
|
|
|
return [float(part) for part in version_parts]
|
|
|
|
|
|
def download_file(url, file_name):
|
|
"""Returns True if download was successful. Raises error if download fails."""
|
|
proc = subprocess.Popen(["curl",
|
|
"-L", "--silent",
|
|
"--retry", "5",
|
|
"--retry-max-time", "600",
|
|
"--max-time", "120",
|
|
"-o", file_name,
|
|
url],
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
proc.communicate()
|
|
error_code = proc.returncode
|
|
if not error_code:
|
|
error_code = proc.wait()
|
|
if not error_code:
|
|
return True
|
|
|
|
raise Exception("Failed to download %s with error %d" % (url, error_code))
|
|
|
|
|
|
class MultiVersionDownloader:
|
|
|
|
def __init__(self, install_dir, link_dir, edition, platform_arch, generic_arch='Linux/x86_64'):
|
|
self.install_dir = install_dir
|
|
self.link_dir = link_dir
|
|
self.edition = edition.lower()
|
|
self.platform_arch = platform_arch.lower().replace('/', '_')
|
|
self.generic_arch = generic_arch.lower().replace('/', '_')
|
|
self._links = None
|
|
self._generic_links = None
|
|
|
|
@property
|
|
def generic_links(self):
|
|
if self._generic_links is None:
|
|
self._links, self._generic_links = self.download_links()
|
|
return self._generic_links
|
|
|
|
@property
|
|
def links(self):
|
|
if self._links is None:
|
|
self._links, self._generic_links = self.download_links()
|
|
return self._links
|
|
|
|
def download_links(self):
|
|
temp_file = tempfile.mktemp()
|
|
download_file("https://downloads.mongodb.org/full.json", temp_file)
|
|
with open(temp_file) as f:
|
|
full_json = json.load(f)
|
|
os.remove(temp_file)
|
|
if 'versions' not in full_json:
|
|
raise Exception("No versions field in JSON: \n" + str(full_json))
|
|
|
|
links = {}
|
|
generic_links = {}
|
|
for json_version in full_json['versions']:
|
|
if 'version' in json_version and 'downloads' in json_version:
|
|
version = json_version['version']
|
|
for download in json_version['downloads']:
|
|
if 'target' in download and 'edition' in download:
|
|
if download['target'].lower() == self.platform_arch and \
|
|
download['edition'].lower() == self.edition:
|
|
links[version] = download['archive']['url']
|
|
elif download['target'].lower() == self.generic_arch and \
|
|
download['edition'].lower() == 'base':
|
|
generic_links[version] = download['archive']['url']
|
|
|
|
return links, generic_links
|
|
|
|
def download_version(self, version):
|
|
|
|
try:
|
|
os.makedirs(self.install_dir)
|
|
except OSError as exc:
|
|
if exc.errno == errno.EEXIST and os.path.isdir(self.install_dir):
|
|
pass
|
|
else: raise
|
|
|
|
urls = []
|
|
requested_version_parts = get_version_parts(version)
|
|
for link_version, link_url in self.links.iteritems():
|
|
link_version_parts = get_version_parts(link_version)
|
|
if link_version_parts[:len(requested_version_parts)] == requested_version_parts:
|
|
# The 'link_version' is a candidate for the requested 'version' if
|
|
# (a) it is a prefix of the requested version, or if
|
|
# (b) it is the "<branchname>-latest" version and the requested version is for a
|
|
# particular major release.
|
|
# This is equivalent to the 'link_version' having components equal to all of the
|
|
# version parts that make up 'version'.
|
|
if "-" in version:
|
|
# The requested 'version' contains a hyphen, so we only consider exact matches
|
|
# to that version.
|
|
if link_version != version:
|
|
continue
|
|
urls.append((link_version, link_url))
|
|
|
|
if len(urls) == 0:
|
|
print >> sys.stderr, ("Cannot find a link for version %s, versions %s found."
|
|
% (version, self.links))
|
|
for ver, generic_url in self.generic_links.iteritems():
|
|
parts = get_version_parts(ver)
|
|
if parts[:len(requested_version_parts)] == requested_version_parts:
|
|
if "-" in version and ver != version:
|
|
continue
|
|
urls.append((ver, generic_url))
|
|
if len(urls) == 0:
|
|
raise Exception(
|
|
"No fall-back generic link available or version %s." % version)
|
|
else:
|
|
print "Falling back to generic architecture."
|
|
|
|
urls.sort(key=lambda (version, _): get_version_parts(version, for_sorting=True))
|
|
full_version = urls[-1][0]
|
|
url = urls[-1][1]
|
|
extract_dir = url.split("/")[-1][:-4]
|
|
file_suffix = os.path.splitext(urlparse.urlparse(url).path)[1]
|
|
|
|
# only download if we don't already have the directory
|
|
already_downloaded = os.path.isdir(os.path.join( self.install_dir, extract_dir))
|
|
if already_downloaded:
|
|
print "Skipping download for version %s (%s) since the dest already exists '%s'" \
|
|
% (version, full_version, extract_dir)
|
|
else:
|
|
print "Downloading data for version %s (%s)..." % (version, full_version)
|
|
print "Download url is %s" % url
|
|
|
|
temp_dir = tempfile.mkdtemp()
|
|
temp_file = tempfile.mktemp(suffix=file_suffix)
|
|
download_file(url, temp_file)
|
|
|
|
print "Uncompressing data for version %s (%s)..." % (version, full_version)
|
|
first_file = ''
|
|
if file_suffix == ".zip":
|
|
# Support .zip downloads, used for Windows binaries.
|
|
with zipfile.ZipFile(temp_file) as zf:
|
|
# Use the name of the root directory in the archive as the name of the directory
|
|
# to extract the binaries into inside 'self.install_dir'. The name of the root
|
|
# directory nearly always matches the parsed URL text, with the exception of
|
|
# versions such as "v3.2-latest" that instead contain the githash.
|
|
first_file = zf.namelist()[0]
|
|
zf.extractall(temp_dir)
|
|
elif file_suffix == ".tgz":
|
|
# Support .tgz downloads, used for Linux binaries.
|
|
with closing(tarfile.open(temp_file, 'r:gz')) as tf:
|
|
# Use the name of the root directory in the archive as the name of the directory
|
|
# to extract the binaries into inside 'self.install_dir'. The name of the root
|
|
# directory nearly always matches the parsed URL text, with the exception of
|
|
# versions such as "v3.2-latest" that instead contain the githash.
|
|
first_file = tf.getnames()[0]
|
|
tf.extractall(path=temp_dir)
|
|
else:
|
|
raise Exception("Unsupported file extension %s" % file_suffix)
|
|
|
|
# Sometimes the zip will contain the root directory as the first file and
|
|
# os.path.dirname() will return ''.
|
|
extract_dir = os.path.dirname(first_file)
|
|
if not extract_dir:
|
|
extract_dir = first_file
|
|
temp_install_dir = os.path.join(temp_dir, extract_dir)
|
|
|
|
# We may not have been able to determine whether we already downloaded the requested
|
|
# version due to the ambiguity in the parsed URL text, so we check for it again using
|
|
# the adjusted 'extract_dir' value.
|
|
already_downloaded = os.path.isdir(os.path.join(self.install_dir, extract_dir))
|
|
if not already_downloaded:
|
|
shutil.move(temp_install_dir, self.install_dir)
|
|
|
|
shutil.rmtree(temp_dir)
|
|
os.remove(temp_file)
|
|
|
|
self.symlink_version(version, os.path.abspath(os.path.join(self.install_dir, extract_dir)))
|
|
|
|
|
|
def symlink_version(self, version, installed_dir):
|
|
|
|
try:
|
|
os.makedirs(self.link_dir)
|
|
except OSError as exc:
|
|
if exc.errno == errno.EEXIST and os.path.isdir(self.link_dir):
|
|
pass
|
|
else: raise
|
|
|
|
for executable in os.listdir(os.path.join(installed_dir, "bin")):
|
|
|
|
executable_name, executable_extension = os.path.splitext(executable)
|
|
link_name = "%s-%s%s" % (executable_name, version, executable_extension)
|
|
|
|
try:
|
|
executable = os.path.join(installed_dir, "bin", executable)
|
|
executable_link = os.path.join(self.link_dir, link_name)
|
|
if os.name == "nt":
|
|
# os.symlink is not supported on Windows, use a direct method instead.
|
|
def symlink_ms(source, link_name):
|
|
import ctypes
|
|
csl = ctypes.windll.kernel32.CreateSymbolicLinkW
|
|
csl.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32)
|
|
csl.restype = ctypes.c_ubyte
|
|
flags = 1 if os.path.isdir(source) else 0
|
|
if csl(link_name, source.replace('/', '\\'), flags) == 0:
|
|
raise ctypes.WinError()
|
|
os.symlink = symlink_ms
|
|
os.symlink(executable, executable_link)
|
|
except OSError as exc:
|
|
if exc.errno == errno.EEXIST:
|
|
pass
|
|
else: raise
|
|
|
|
|
|
CL_HELP_MESSAGE = \
|
|
"""
|
|
Downloads and installs particular mongodb versions (each binary is renamed to include its version)
|
|
into an install directory and symlinks the binaries with versions to another directory. This script
|
|
supports community and enterprise builds.
|
|
|
|
Usage: setup_multiversion_mongodb.py INSTALL_DIR LINK_DIR EDITION PLATFORM_AND_ARCH VERSION1 [VERSION2 VERSION3 ...]
|
|
|
|
EDITION is one of the following:
|
|
base (generic community builds)
|
|
enterprise
|
|
targeted (platform specific community builds, includes SSL)
|
|
PLATFORM_AND_ARCH can be specified with just a platform, i.e., OSX, if it is supported.
|
|
|
|
Ex: setup_multiversion_mongodb.py ./install ./link base "Linux/x86_64" "2.0.6" "2.0.3-rc0" "2.0" "2.2" "2.3"
|
|
Ex: setup_multiversion_mongodb.py ./install ./link enterprise "OSX" "2.4" "2.2"
|
|
|
|
After running the script you will have a directory structure like this:
|
|
./install/[mongodb-osx-x86_64-2.4.9, mongodb-osx-x86_64-2.2.7]
|
|
./link/[mongod-2.4.9, mongod-2.2.7, mongo-2.4.9...]
|
|
|
|
You should then add ./link/ to your path so multi-version tests will work.
|
|
|
|
Note: If "rc" is included in the version name, we'll use the exact rc, otherwise we'll pull the highest non-rc
|
|
version compatible with the version specified.
|
|
"""
|
|
|
|
def parse_cl_args(args):
|
|
|
|
def raise_exception(msg):
|
|
print CL_HELP_MESSAGE
|
|
raise Exception(msg)
|
|
|
|
if len(args) == 0: raise_exception("Missing INSTALL_DIR")
|
|
|
|
install_dir = args[0]
|
|
|
|
args = args[1:]
|
|
if len(args) == 0: raise_exception("Missing LINK_DIR")
|
|
|
|
link_dir = args[0]
|
|
|
|
args = args[1:]
|
|
if len(args) == 0: raise_exception("Missing EDITION")
|
|
|
|
edition = args[0]
|
|
if edition not in ['base', 'enterprise', 'targeted']:
|
|
raise Exception("Unsupported edition %s" % edition)
|
|
|
|
args = args[1:]
|
|
if len(args) == 0: raise_exception("Missing PLATFORM_AND_ARCH")
|
|
|
|
platform_arch = args[0]
|
|
|
|
args = args[1:]
|
|
|
|
if len(args) == 0: raise_exception("Missing VERSION1")
|
|
|
|
versions = args
|
|
|
|
return (MultiVersionDownloader(install_dir, link_dir, edition, platform_arch), versions)
|
|
|
|
def main():
|
|
|
|
# Listen for SIGUSR1 and dump stack if received.
|
|
try:
|
|
signal.signal(signal.SIGUSR1, dump_stacks)
|
|
except AttributeError:
|
|
print "Cannot catch signals on Windows"
|
|
|
|
downloader, versions = parse_cl_args(sys.argv[1:])
|
|
|
|
for version in versions:
|
|
downloader.download_version(version)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|