mirrors/mongodb
mirrors/mongodb
0
0
Fork 0
mirror of https://github.com/mongodb/mongo.git synced 2024-12-01 09:32:32 +01:00
Code Issues Releases Activity

SERVER-32600 Add architecture parameter to setup_multiversion_mongodb.py to ensure it downloads the proper binary

Browse Source
This commit is contained in:
Jonathan Abrahams 2018-03-01 10:02:40 -05:00
parent 571e035928
commit 4644d79d8c
2 changed files with 135 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

156
buildscripts/setup_multiversion_mongodb.py
View File

@ -23,7 +23,6 @@ import zipfile
import requests
import requests.exceptions
def dump_stacks(_signal_num, _frame):
"""Dump stacks when SIGUSR1 is received."""
print("======================================")
@ -103,7 +102,8 @@ def download_file(url, file_name, download_retries=5):
download_retries -= 1
if download_retries == 0:
raise Exception("Downloaded file size ({} bytes) doesn't match content length"
"({} bytes) for URL {}".format(file_size, url_content_length, url))
"({} bytes) for URL {}".format(
file_size, url_content_length, url))
continue
return True
@ -118,13 +118,16 @@ class MultiVersionDownloader(object):
install_dir,
link_dir,
edition,
platform_arch,
platform,
architecture,
use_latest=False):
self.install_dir = install_dir
self.link_dir = link_dir
self.edition = edition.lower()
self.platform_arch = platform_arch.lower().replace("/", "_")
self.generic_arch = "linux_x86_64"
self.platform = platform.lower()
self.architecture = architecture.lower()
self.generic_platform = "linux"
self.generic_architecture = "x86_64"
self.use_latest = use_latest
self._links = None
self._generic_links = None
@ -162,22 +165,35 @@ class MultiVersionDownloader(object):
links = {}
generic_links = {}
# The generic target contains a platform and architecture.
generic_target = "{}_{}".format(self.generic_platform, self.generic_architecture)
for json_version in full_json["versions"]:
if "version" in json_version and 'downloads' in json_version:
version = json_version["version"]
for download in json_version["downloads"]:
if "target" in download and "edition" in download:
if download["target"].lower() == self.platform_arch and \
download["edition"].lower() == self.edition:
links[version] = download["archive"]["url"]
elif download["target"].lower() == self.generic_arch and \
download["edition"].lower() == "base":
generic_links[version] = download["archive"]["url"]
if "version" not in json_version or 'downloads' not in json_version:
continue
version = json_version["version"]
for download in json_version["downloads"]:
if "target" not in download or "edition" not in download:
continue
if (download["target"].lower() == self.platform and
download["arch"].lower() == self.architecture and
download["edition"].lower() == self.edition):
links[version] = download["archive"]["url"]
elif (download["target"].lower() == generic_target and
download["edition"].lower() == "base"):
generic_links[version] = download["archive"]["url"]
return links, generic_links
def download_install(self, version):
"""Downloads and installs the version specified."""
download_file = self.download_version(version)
if download_file:
installed_dir = self.uncompress_download(download_file)
self.symlink_version(version, installed_dir)
def download_version(self, version):
"""Downloads the version specified."""
"""Downloads the version specified and returns file location.
If no download occurs, file location is None."""
try:
os.makedirs(self.install_dir)
@ -233,8 +249,8 @@ class MultiVersionDownloader(object):
if already_downloaded:
print("Skipping download for version {} ({}) since the dest already exists '{}'"
.format(version, full_version, extract_dir))
return None
else:
temp_dir = tempfile.mkdtemp()
temp_file = tempfile.mktemp(suffix=file_suffix)
latest_downloaded = False
@ -258,48 +274,54 @@ class MultiVersionDownloader(object):
print("Downloading data for version {} ({})...".format(version, full_version))
print("Download url is {}".format(url))
download_file(url, temp_file)
return temp_file
print("Uncompressing data for version {} ({})...".format(version, full_version))
first_file = ""
if file_suffix == ".zip":
# Support .zip downloads, used for Windows binaries.
with zipfile.ZipFile(temp_file) as zip_handle:
# Use the name of the root directory in the archive as the name of the directory
# to extract the binaries into inside 'self.install_dir'. The name of the root
# directory nearly always matches the parsed URL text, with the exception of
# versions such as "v3.2-latest" that instead contain the githash.
first_file = zip_handle.namelist()[0]
zip_handle.extractall(temp_dir)
elif file_suffix == ".tgz":
# Support .tgz downloads, used for Linux binaries.
with contextlib.closing(tarfile.open(temp_file, "r:gz")) as tar_handle:
# Use the name of the root directory in the archive as the name of the directory
# to extract the binaries into inside 'self.install_dir'. The name of the root
# directory nearly always matches the parsed URL text, with the exception of
# versions such as "v3.2-latest" that instead contain the githash.
first_file = tar_handle.getnames()[0]
tar_handle.extractall(path=temp_dir)
else:
raise Exception("Unsupported file extension {}".format(file_suffix))
def uncompress_download(self, download_file):
"""Downloads the version specified and returns root of ."""
# Sometimes the zip will contain the root directory as the first file and
# os.path.dirname() will return ''.
extract_dir = os.path.dirname(first_file)
if not extract_dir:
extract_dir = first_file
temp_install_dir = os.path.join(temp_dir, extract_dir)
print("Uncompressing data to {}...".format(self.install_dir))
first_file = ""
temp_dir = tempfile.mkdtemp()
_, file_suffix = os.path.splitext(download_file)
if file_suffix == ".zip":
# Support .zip downloads, used for Windows binaries.
with zipfile.ZipFile(download_file) as zip_handle:
# Use the name of the root directory in the archive as the name of the directory
# to extract the binaries into inside 'self.install_dir'. The name of the root
# directory nearly always matches the parsed URL text, with the exception of
# versions such as "v3.2-latest" that instead contain the githash.
first_file = zip_handle.namelist()[0]
zip_handle.extractall(temp_dir)
elif file_suffix == ".tgz":
# Support .tgz downloads, used for Linux binaries.
with contextlib.closing(tarfile.open(download_file, "r:gz")) as tar_handle:
# Use the name of the root directory in the archive as the name of the directory
# to extract the binaries into inside 'self.install_dir'. The name of the root
# directory nearly always matches the parsed URL text, with the exception of
# versions such as "v3.2-latest" that instead contain the githash.
first_file = tar_handle.getnames()[0]
tar_handle.extractall(path=temp_dir)
else:
raise Exception("Unsupported file extension {}".format(file_suffix))
# We may not have been able to determine whether we already downloaded the requested
# version due to the ambiguity in the parsed URL text, so we check for it again using
# the adjusted 'extract_dir' value.
already_downloaded = os.path.isdir(os.path.join(self.install_dir, extract_dir))
if not already_downloaded:
shutil.move(temp_install_dir, self.install_dir)
# Sometimes the zip will contain the root directory as the first file and
# os.path.dirname() will return ''.
extract_dir = os.path.dirname(first_file)
if not extract_dir:
extract_dir = first_file
temp_install_dir = os.path.join(temp_dir, extract_dir)
shutil.rmtree(temp_dir)
os.remove(temp_file)
# We may not have been able to determine whether we already downloaded the requested
# version due to the ambiguity in the parsed URL text, so we check for it again using
# the adjusted 'extract_dir' value.
already_downloaded = os.path.isdir(os.path.join(self.install_dir, extract_dir))
if not already_downloaded:
shutil.move(temp_install_dir, self.install_dir)
self.symlink_version(version, os.path.abspath(os.path.join(self.install_dir, extract_dir)))
shutil.rmtree(temp_dir)
os.remove(download_file)
return os.path.abspath(os.path.join(self.install_dir, extract_dir))
def symlink_version(self, version, installed_dir):
"""Symlinks the binaries in the 'installed_dir' to the 'link_dir.'"""
@ -359,12 +381,13 @@ Usage: setup_multiversion_mongodb.py [options] ver1 [vers2 ...]
Ex: setup_multiversion_mongodb.py --installDir ./install
--linkDir ./link
--edition base
--platformArchitecture Linux/x86_64 2.0.6 2.0.3-rc0
--platform Linux 2.0.6 2.0.3-rc0
--architecture x86_64
2.0 2.2 2.3
Ex: setup_multiversion_mongodb.py --installDir ./install
--linkDir ./link
--edition enterprise
--platformArchitecture osx
--platform osx
2.4 2.2
After running the script you will have a directory structure like this:
@ -393,12 +416,16 @@ we'll pull the highest non-rc version compatible with the version specified.
help="Edition of the build to download, choose from {}, [default:"
" '%default'].".format(editions),
default="base")
parser.add_option("-p", "--platformArchitecture",
dest="platform_arch",
help="Platform/architecture to download. The architecture is not required."
" [REQUIRED]. Examples include: 'linux/x86_64', 'osx', 'rhel62',"
" 'windows/x86_64-2008plus-ssl'.",
parser.add_option("-p", "--platform",
dest="platform",
help="Platform to download [REQUIRED]. Examples include: 'linux',"
" 'osx', 'rhel62', 'windows'.",
default=None)
parser.add_option("-a", "--architecture",
dest="architecture",
help="Architecture to download, [default: '%default']. Examples include:"
" 'arm64', 'ppc64le', 's390x' and 'x86_64'.",
default="x86_64")
parser.add_option("-u", "--useLatest",
dest="use_latest",
action="store_true",
@ -415,7 +442,7 @@ we'll pull the highest non-rc version compatible with the version specified.
if (not versions or
not options.install_dir or
not options.link_dir or
not options.platform_arch):
not options.platform):
parser.print_help()
parser.exit(1)
@ -423,11 +450,12 @@ we'll pull the highest non-rc version compatible with the version specified.
options.install_dir,
options.link_dir,
options.edition,
options.platform_arch,
options.platform,
options.architecture,
options.use_latest)
for version in versions:
downloader.download_version(version)
downloader.download_install(version)
if __name__ == "__main__":

94
etc/evergreen.yml
View File

@ -1148,11 +1148,12 @@ functions:
pip install 'pyOpenSSL ; sys_platform == "win32" or sys_platform == "cygwin"'
rm -rf /data/install /data/multiversion
$python buildscripts/setup_multiversion_mongodb.py \
--installDir /data/install \
--linkDir /data/multiversion \
--edition ${multiversion_edition|base} \
--platformArchitecture ${multiversion_platform_arch|linux/x86_64} \
$python buildscripts/setup_multiversion_mongodb.py \
--installDir /data/install \
--linkDir /data/multiversion \
--edition ${multiversion_edition|base} \
--platform ${multiversion_platform|linux} \
--architecture ${multiversion_architecture|x86_64} \
--useLatest 3.0 3.2 3.4 3.6
"do snmp setup" :
@ -4993,8 +4994,8 @@ buildvariants:
push_name: linux
push_arch: x86_64-ubuntu1204
compile_flags: --ssl MONGO_DISTMOD=ubuntu1204 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars
multiversion_platform_arch: "ubuntu1204"
multiversion_edition: "targeted"
multiversion_platform: ubuntu1204
multiversion_edition: targeted
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
has_packages: true
packager_script: packager.py
@ -5075,8 +5076,8 @@ buildvariants:
push_name: linux
push_arch: x86_64-ubuntu1404
compile_flags: --ssl MONGO_DISTMOD=ubuntu1404 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars
multiversion_platform_arch: "ubuntu1404"
multiversion_edition: "targeted"
multiversion_platform: ubuntu1404
multiversion_edition: targeted
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
has_packages: true
packager_script: packager.py
@ -5185,11 +5186,8 @@ buildvariants:
push_arch: x86_64-ubuntu1604
lang_environment: LANG=C
compile_flags: --ssl MONGO_DISTMOD=ubuntu1604 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars
# TODO SERVER-32600: Re-enable targeted downloads after setup_multiversion_mongodb.py is
# guaranteed to download x86_64 binaries for this platform.
#
# multiversion_platform_arch: "ubuntu1604"
# multiversion_edition: "targeted"
multiversion_platform: ubuntu1604
multiversion_edition: targeted
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
has_packages: true
packager_script: packager.py
@ -5651,8 +5649,8 @@ buildvariants:
push_name: linux
push_arch: x86_64-amazon
compile_flags: --ssl MONGO_DISTMOD=amazon -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars
multiversion_platform_arch: "amzn64"
multiversion_edition: "enterprise"
multiversion_platform: amzn64
multiversion_edition: enterprise
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
has_packages: true
packager_script: packager.py
@ -6067,7 +6065,7 @@ buildvariants:
push_bucket: downloads.mongodb.org
push_name: win32
push_arch: x86_64-2008plus-ssl
multiversion_platform_arch: windows/x86_64-2008plus-ssl
multiversion_platform: windows_x86_64-2008plus-ssl
msi_target: msi
content_type: application/zip
compile_flags: --release --ssl MONGO_DISTMOD=2008plus-ssl CPPPATH="c:/openssl/include" LIBPATH="c:/openssl/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --dynamic-windows --win-version-min=ws08r2
@ -6213,7 +6211,7 @@ buildvariants:
gorootvars: CGO_CPPFLAGS=-I/opt/mongodbtoolchain/v2/include CGO_CFLAGS=-mmacosx-version-min=10.10 CGO_LDFLAGS=-mmacosx-version-min=10.10
compile_env: DEVELOPER_DIR=/Applications/Xcode8.3.app
compile_flags: --ssl -j$(sysctl -n hw.logicalcpu) --release --libc++ CCFLAGS="-mmacosx-version-min=10.10" LINKFLAGS="-mmacosx-version-min=10.10" CPPPATH=/opt/mongodbtoolchain/v2/include
multiversion_platform_arch: "osx-ssl"
multiversion_platform: osx-ssl
num_jobs_available: 1
build_mongoreplay: true
tasks:
@ -6469,8 +6467,8 @@ buildvariants:
push_arch: x86_64-enterprise-rhel62
rlp_environment: MONGOD_UNITTEST_RLP_LANGUAGE_TEST_BTROOT=/opt/basis
compile_flags: --ssl MONGO_DISTMOD=rhel62 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars CPPPATH="/opt/basis/rlp/rlp/include /opt/basis/rlp/utilities/include" --use-basis-tech-rosette-linguistics-platform=on
multiversion_platform_arch: "rhel62"
multiversion_edition: "targeted"
multiversion_platform: rhel62
multiversion_edition: targeted
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
has_packages: true
packager_script: packager-enterprise.py
@ -6664,8 +6662,8 @@ buildvariants:
gorootvars: GOROOT=/opt/go PATH="/opt/go/bin:$PATH"
tooltags: "-tags 'ssl sasl'"
compile_flags: --dbg=on --gcov --ssl MONGO_DISTMOD=rhel62 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_gcc.vars
multiversion_platform_arch: "rhel62"
multiversion_edition: "targeted"
multiversion_platform: rhel62
multiversion_edition: targeted
num_jobs_available: $(($(grep -c ^processor /proc/cpuinfo) / 2)) # Avoid starting too many mongod's
# The gcov instrumentation saves the path the .gcno files were created in as the default path
# for the .gcda files. In Evergreen the path will start with /data/mci/[Hashed ID]/src/... where
@ -6861,8 +6859,8 @@ buildvariants:
push_name: linux
push_arch: x86_64-rhel62
compile_flags: --ssl MONGO_DISTMOD=rhel62 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars
multiversion_platform_arch: "rhel62"
multiversion_edition: "targeted"
multiversion_platform: rhel62
multiversion_edition: targeted
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
has_packages: true
packager_script: packager.py
@ -6944,8 +6942,8 @@ buildvariants:
push_name: linux
push_arch: x86_64-rhel70
compile_flags: --ssl MONGO_DISTMOD=rhel70 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars
multiversion_platform_arch: "rhel70"
multiversion_edition: "targeted"
multiversion_platform: rhel70
multiversion_edition: targeted
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
has_packages: true
packager_script: packager.py
@ -7724,8 +7722,8 @@ buildvariants:
push_arch: x86_64-suse12
compile_flags: --ssl MONGO_DISTMOD=suse12 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
multiversion_platform_arch: "suse12"
multiversion_edition: "targeted"
multiversion_platform: suse12
multiversion_edition: targeted
has_packages: true
packager_script: packager.py
packager_arch: x86_64
@ -7919,8 +7917,8 @@ buildvariants:
push_name: linux
push_arch: x86_64-debian71
compile_flags: --ssl MONGO_DISTMOD=debian71 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars
multiversion_platform_arch: "debian71"
multiversion_edition: "targeted"
multiversion_platform: debian71
multiversion_edition: targeted
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
has_packages: true
packager_script: packager.py
@ -8005,8 +8003,8 @@ buildvariants:
push_name: linux
push_arch: x86_64-debian81
compile_flags: --ssl MONGO_DISTMOD=debian81 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars
multiversion_platform_arch: "debian81"
multiversion_edition: "targeted"
multiversion_platform: debian81
multiversion_edition: targeted
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
has_packages: true
packager_script: packager.py
@ -8144,8 +8142,8 @@ buildvariants:
push_name: linux
push_arch: x86_64-debian92
compile_flags: --ssl MONGO_DISTMOD=debian92 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars
multiversion_platform_arch: "debian92"
multiversion_edition: "targeted"
multiversion_platform: debian92
multiversion_edition: targeted
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
has_packages: true
packager_script: packager.py
@ -8232,8 +8230,8 @@ buildvariants:
rlp_environment: MONGOD_UNITTEST_RLP_LANGUAGE_TEST_BTROOT=/opt/basis
test_flags: --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling,requires_mmapv1
compile_flags: --ssl MONGO_DISTMOD=rhel62 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_gcc.vars CPPPATH="/opt/basis/rlp/rlp/include /opt/basis/rlp/utilities/include" --use-basis-tech-rosette-linguistics-platform=on
multiversion_platform_arch: "rhel62"
multiversion_edition: "enterprise"
multiversion_platform: rhel62
multiversion_edition: enterprise
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
use_scons_cache: true
build_mongoreplay: true
@ -8731,11 +8729,8 @@ buildvariants:
lang_environment: LANG=C
san_options: LSAN_OPTIONS="suppressions=etc/lsan.suppressions:report_objects=1" ASAN_OPTIONS=detect_leaks=1:check_initialization_order=true:strict_init_order=true:abort_on_error=1:disable_coredump=0:handle_abort=0:handle_segv=0:handle_sigbus=0:handle_sigill=0:handle_sigfpe=0
compile_flags: --variables-files=etc/scons/mongodbtoolchain_clang.vars --dbg=on --opt=on --allocator=system --sanitize=address --ssl -j$(grep -c ^processor /proc/cpuinfo) --nostrip
# TODO SERVER-32600: Re-enable targeted downloads after setup_multiversion_mongodb.py is
# guaranteed to download x86_64 binaries for this platform.
#
# multiversion_platform_arch: "ubuntu1604"
# multiversion_edition: "targeted"
multiversion_platform: ubuntu1604
multiversion_edition: targeted
num_jobs_available: $(($(grep -c ^processor /proc/cpuinfo) / 3)) # Avoid starting too many mongod's under ASAN build.
build_mongoreplay: true
hang_analyzer_dump_core: false
@ -8889,11 +8884,8 @@ buildvariants:
lang_environment: LANG=C
san_options: UBSAN_OPTIONS="print_stacktrace=1:handle_abort=0:handle_segv=0:handle_sigbus=0:handle_sigill=0:handle_sigfpe=0"
compile_flags: --variables-files=etc/scons/mongodbtoolchain_clang.vars --dbg=on --opt=on --allocator=system --sanitize=undefined --ssl -j$(grep -c ^processor /proc/cpuinfo) --nostrip
# TODO SERVER-32600: Re-enable targeted downloads after setup_multiversion_mongodb.py is
# guaranteed to download x86_64 binaries for this platform.
#
# multiversion_platform_arch: "ubuntu1604"
# multiversion_edition: "targeted"
multiversion_platform: ubuntu1604
multiversion_edition: targeted
num_jobs_available: $(($(grep -c ^processor /proc/cpuinfo) / 3)) # Avoid starting too many mongod's under UBSAN build.
build_mongoreplay: true
tasks:
@ -9170,7 +9162,7 @@ buildvariants:
--variables-files=etc/scons/mongodbtoolchain_gcc.vars
CPPPATH="/opt/basis/rlp/rlp/include /opt/basis/rlp/utilities/include"
--use-basis-tech-rosette-linguistics-platform=on
multiversion_platform_arch: rhel62
multiversion_platform: rhel62
multiversion_edition: targeted
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
has_packages: true
@ -9280,8 +9272,8 @@ buildvariants:
gorootvars: >-
PATH="/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:/cygdrive/c/sasl/:$PATH"
CGO_CFLAGS="-D_WIN32_WINNT=0x0601 -DNTDDI_VERSION=0x06010000"
multiversion_platform_arch: windows
multiversion_edition: "enterprise"
multiversion_platform: windows
multiversion_edition: enterprise
msi_target: msi
content_type: application/zip
compile_flags: >-
@ -9405,8 +9397,8 @@ buildvariants:
compile_flags: >-
--ssl -j$(sysctl -n hw.logicalcpu) --release --libc++ CCFLAGS="-mmacosx-version-min=10.10"
LINKFLAGS="-mmacosx-version-min=10.10" CPPPATH=/opt/mongodbtoolchain/v2/include
multiversion_platform_arch: "osx"
multiversion_edition: "enterprise"
multiversion_platform: osx
multiversion_edition: enterprise
num_jobs_available: 1
build_mongoreplay: true
tasks: