mirror of
https://github.com/mongodb/mongo.git
synced 2024-12-01 09:32:32 +01:00
SERVER-33661 Constrain link parallelism for compile_all and compile_unittests
This commit is contained in:
Andrew Morrow
parent
5cdc587610
commit
e88d29393f
41
SConstruct
41
SConstruct
@ -508,6 +508,11 @@ add_option('msvc-debugging-format',
|
||||
type='choice',
|
||||
)
|
||||
|
||||
add_option('jlink',
|
||||
help="Limit link concurrency to given value",
|
||||
nargs=1,
|
||||
type=int)
|
||||
|
||||
try:
|
||||
with open("version.json", "r") as version_fp:
|
||||
version_data = json.load(version_fp)
|
||||
@ -3413,6 +3418,42 @@ env.Alias("distsrc-tgz", env.GZip(
|
||||
env.Alias("distsrc-zip", env.DistSrc("mongodb-src-${MONGO_VERSION}.zip"))
|
||||
env.Alias("distsrc", "distsrc-tgz")
|
||||
|
||||
# Do this as close to last as possible before reading SConscripts, so
|
||||
# that any tools that may have injected other things via emitters are included
|
||||
# among the side effect adornments.
|
||||
#
|
||||
# TODO: Move this to a tool.
|
||||
if has_option('jlink'):
|
||||
jlink = get_option('jlink')
|
||||
if jlink < 1:
|
||||
env.FatalError("The argument to jlink must be a positive integer")
|
||||
|
||||
target_builders = ['Program', 'SharedLibrary', 'LoadableModule']
|
||||
|
||||
# A bound map of stream (as in stream of work) name to side-effect
|
||||
# file. Since SCons will not allow tasks with a shared side-effect
|
||||
# to execute concurrently, this gives us a way to limit link jobs
|
||||
# independently of overall SCons concurrency.
|
||||
jlink_stream_map = dict()
|
||||
|
||||
def jlink_emitter(target, source, env):
|
||||
name = str(target[0])
|
||||
se_name = "#jlink-stream" + str(hash(name) % jlink)
|
||||
se_node = jlink_stream_map.get(se_name, None)
|
||||
if not se_node:
|
||||
se_node = env.Entry(se_name)
|
||||
# This may not be necessary, but why chance it
|
||||
env.NoCache(se_node)
|
||||
jlink_stream_map[se_name] = se_node
|
||||
env.SideEffect(se_node, target)
|
||||
return (target, source)
|
||||
|
||||
for target_builder in target_builders:
|
||||
builder = env['BUILDERS'][target_builder]
|
||||
base_emitter = builder.emitter
|
||||
new_emitter = SCons.Builder.ListEmitter([base_emitter, jlink_emitter])
|
||||
builder.emitter = new_emitter
|
||||
|
||||
env.SConscript(
|
||||
dirs=[
|
||||
'src',
|
||||
|
||||
@ -969,9 +969,9 @@ functions:
|
||||
rm -rf ${install_directory|/data/mongo-install-directory}
|
||||
|
||||
extra_args=""
|
||||
if [ "${reduce_scons_compile_jobs_available|false}" = "true" ] && [ -n "${num_scons_compile_jobs_available|}" ]; then
|
||||
echo "Changing SCons to run with --jobs=${num_scons_compile_jobs_available|}"
|
||||
extra_args="$extra_args --jobs=${num_scons_compile_jobs_available|}"
|
||||
if [ -n "${num_scons_link_jobs_available|}" ]; then
|
||||
echo "Changing SCons to run with --jlink=${num_scons_link_jobs_available|}"
|
||||
extra_args="$extra_args --jlink=${num_scons_link_jobs_available|}"
|
||||
fi
|
||||
|
||||
${activate_virtualenv}
|
||||
@ -2849,7 +2849,6 @@ tasks:
|
||||
- func: "scons compile"
|
||||
vars:
|
||||
targets: all
|
||||
reduce_scons_compile_jobs_available: true
|
||||
task_compile_flags: >-
|
||||
--use-new-tools
|
||||
--build-mongoreplay="${build_mongoreplay}"
|
||||
@ -2872,7 +2871,6 @@ tasks:
|
||||
- func: "scons compile"
|
||||
vars:
|
||||
targets: unittests
|
||||
reduce_scons_compile_jobs_available: true
|
||||
task_compile_flags: >-
|
||||
--detect-odr-violations
|
||||
|
||||
@ -7622,7 +7620,7 @@ buildvariants:
|
||||
compile_flags: --ssl MONGO_DISTMOD=amazon2 --release -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_gcc.vars
|
||||
# We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to
|
||||
# spawning a large number of linker processes.
|
||||
num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
|
||||
has_packages: true
|
||||
packager_script: packager_enterprise.py
|
||||
@ -7699,7 +7697,7 @@ buildvariants:
|
||||
compile_flags: --ssl MONGO_DISTMOD=amazon2 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars
|
||||
# We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to
|
||||
# spawning a large number of linker processes.
|
||||
num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
multiversion_platform: amazon
|
||||
multiversion_edition: targeted
|
||||
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
|
||||
@ -7798,7 +7796,7 @@ buildvariants:
|
||||
compile_flags: --dbg=on --opt=on --win-version-min=ws08r2 -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) MONGO_DISTMOD=2008plus
|
||||
# We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to
|
||||
# spawning a large number of linker processes.
|
||||
num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
python: python
|
||||
ext: zip
|
||||
use_scons_cache: true
|
||||
@ -7940,7 +7938,7 @@ buildvariants:
|
||||
compile_flags: --release --ssl MONGO_DISTMOD=windows-64 CPPPATH="c:/openssl/include c:/sasl/include c:/snmp/include c:/curl/include" LIBPATH="c:/openssl/lib c:/sasl/lib c:/snmp/lib c:/curl/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --dynamic-windows --win-version-min=ws08r2
|
||||
# We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to
|
||||
# spawning a large number of linker processes.
|
||||
num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
python: python
|
||||
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
|
||||
ext: zip
|
||||
@ -8312,7 +8310,7 @@ buildvariants:
|
||||
compile_flags: --release --ssl MONGO_DISTMOD=windows-64 CPPPATH="c:/openssl/include c:/sasl/include c:/snmp/include c:/curl/include" LIBPATH="c:/openssl/lib c:/sasl/lib c:/snmp/lib c:/curl/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --dynamic-windows --win-version-min=ws08r2
|
||||
# We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to
|
||||
# spawning a large number of linker processes.
|
||||
num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
python: python
|
||||
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
|
||||
ext: zip
|
||||
@ -8336,7 +8334,7 @@ buildvariants:
|
||||
compile_flags: --release --ssl --ssl-provider=openssl MONGO_DISTMOD=windows-64 CPPPATH="c:/openssl/include c:/sasl/include c:/snmp/include c:/curl/include" LIBPATH="c:/openssl/lib c:/sasl/lib c:/snmp/lib c:/curl/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --dynamic-windows --win-version-min=ws08r2
|
||||
# We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to
|
||||
# spawning a large number of linker processes.
|
||||
num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
python: python
|
||||
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
|
||||
ext: zip
|
||||
@ -8365,7 +8363,7 @@ buildvariants:
|
||||
compile_flags: --release --ssl MONGO_DISTMOD=windows-64 CPPPATH="c:/openssl/include c:/sasl/include c:/snmp/include c:/curl/include" LIBPATH="c:/openssl/lib c:/sasl/lib c:/snmp/lib c:/curl/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --dynamic-windows --win-version-min=ws08r2
|
||||
# We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to
|
||||
# spawning a large number of linker processes.
|
||||
num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
python: python
|
||||
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
|
||||
ext: zip
|
||||
@ -8401,7 +8399,7 @@ buildvariants:
|
||||
compile_flags: --release --ssl MONGO_DISTMOD=windows-64 CPPPATH="c:/openssl/include c:/sasl/include c:/snmp/include c:/curl/include" LIBPATH="c:/openssl/lib c:/sasl/lib c:/snmp/lib c:/curl/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --dynamic-windows --win-version-min=ws08r2
|
||||
# We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to
|
||||
# spawning a large number of linker processes.
|
||||
num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
python: python
|
||||
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
|
||||
test_flags: --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling,requires_mmapv1
|
||||
@ -8471,7 +8469,7 @@ buildvariants:
|
||||
compile_flags: --release --ssl MONGO_DISTMOD=2008plus-ssl CPPPATH="c:/openssl/include" LIBPATH="c:/openssl/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --dynamic-windows --win-version-min=ws08r2
|
||||
# We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to
|
||||
# spawning a large number of linker processes.
|
||||
num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
python: python
|
||||
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
|
||||
ext: zip
|
||||
@ -8583,7 +8581,7 @@ buildvariants:
|
||||
compile_flags: --dbg=on --opt=off --ssl MONGO_DISTMOD=2008plus CPPPATH="c:/openssl/include c:/sasl/include c:/snmp/include c:/curl/include" LIBPATH="c:/openssl/lib c:/sasl/lib c:/snmp/lib c:/curl/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --dynamic-windows --win-version-min=ws08r2
|
||||
# We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to
|
||||
# spawning a large number of linker processes.
|
||||
num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
python: python
|
||||
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
|
||||
ext: zip
|
||||
@ -11186,7 +11184,7 @@ buildvariants:
|
||||
compile_flags: --ssl MONGO_DISTMOD=debian92 --release -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_gcc.vars
|
||||
# We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to
|
||||
# spawning a large number of linker processes.
|
||||
num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
|
||||
has_packages: true
|
||||
packager_script: packager_enterprise.py
|
||||
@ -11241,7 +11239,7 @@ buildvariants:
|
||||
compile_flags: --ssl MONGO_DISTMOD=debian92 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars
|
||||
# We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to
|
||||
# spawning a large number of linker processes.
|
||||
num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
multiversion_platform: debian92
|
||||
multiversion_edition: targeted
|
||||
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
|
||||
@ -12560,7 +12558,7 @@ buildvariants:
|
||||
c:/sasl/include c:/snmp/include c:/curl/include" LIBPATH="c:/openssl/lib c:/sasl/lib
|
||||
c:/snmp/lib c:/curl/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --dynamic-windows
|
||||
--win-version-min=ws08r2
|
||||
num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
|
||||
python: python
|
||||
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
|
||||
ext: zip
|
||||
|
||||
Loading…
Reference in New Issue
Block a user