mirror of
https://github.com/mongodb/mongo.git
synced 2024-12-01 09:32:32 +01:00
95 lines
3.1 KiB
Python
95 lines
3.1 KiB
Python
import SCons
|
|
|
|
import re
|
|
|
|
task_limiter_patterns = {}
|
|
|
|
|
|
def setup_task_limiter(env, name, concurrency_ratio=0.75, builders=None, source_file_regex='.*',
|
|
target_file_regex='.*'):
|
|
|
|
global task_limiter_patterns
|
|
|
|
task_limiter_patterns[name] = {}
|
|
task_limiter_patterns[name]['source'] = re.compile(source_file_regex)
|
|
task_limiter_patterns[name]['target'] = re.compile(target_file_regex)
|
|
|
|
# We need to convert the ratio value into a int that corrlates to a specific
|
|
# number of concurrent jobs allowed
|
|
concurrency_ratio = float(concurrency_ratio)
|
|
if concurrency_ratio <= 0.0:
|
|
env.FatalError(
|
|
f"The concurrency ratio for {name} must be a positive, got {max_concurrency}")
|
|
|
|
if concurrency_ratio > 1.0:
|
|
concurrency_ratio = 1.0
|
|
|
|
max_concurrency = env.GetOption('num_jobs') * concurrency_ratio
|
|
max_concurrency = round(max_concurrency)
|
|
if max_concurrency < 1.0:
|
|
max_concurrency = 1.0
|
|
|
|
max_concurrency = int(max_concurrency)
|
|
|
|
# A bound map of stream (as in stream of work) name to side-effect
|
|
# file. Since SCons will not allow tasks with a shared side-effect
|
|
# to execute concurrently, this gives us a way to limit link jobs
|
|
# independently of overall SCons concurrency.
|
|
concurrent_stream_map = dict()
|
|
|
|
def task_limiter_emitter(target, source, env):
|
|
global task_limiter_patterns
|
|
nonlocal name
|
|
|
|
matched = False
|
|
for s_file in source:
|
|
if re.search(task_limiter_patterns[name]['source'], s_file.path):
|
|
matched = True
|
|
break
|
|
|
|
if not matched:
|
|
for t_file in target:
|
|
if re.search(task_limiter_patterns[name]['target'], t_file.path):
|
|
matched = True
|
|
break
|
|
if matched:
|
|
se_name = f"#{name}-stream{hash(str(target[0])) % max_concurrency}"
|
|
se_node = concurrent_stream_map.get(se_name, None)
|
|
if not se_node:
|
|
se_node = env.Entry(se_name)
|
|
# This may not be necessary, but why chance it
|
|
env.NoCache(se_node)
|
|
concurrent_stream_map[se_name] = se_node
|
|
env.SideEffect(se_node, target)
|
|
|
|
return (target, source)
|
|
|
|
if isinstance(builders, dict):
|
|
for target_builder, suffixes in builders.items():
|
|
builder = env['BUILDERS'][target_builder]
|
|
emitterdict = builder.builder.emitter
|
|
for suffix in emitterdict.keys():
|
|
if not suffix in suffixes:
|
|
continue
|
|
base = emitterdict[suffix]
|
|
emitterdict[suffix] = SCons.Builder.ListEmitter([
|
|
base,
|
|
task_limiter_emitter,
|
|
])
|
|
else:
|
|
for target_builder in builders:
|
|
builder = env['BUILDERS'][target_builder]
|
|
base_emitter = builder.emitter
|
|
new_emitter = SCons.Builder.ListEmitter([base_emitter, task_limiter_emitter])
|
|
builder.emitter = new_emitter
|
|
|
|
return max_concurrency
|
|
|
|
|
|
def exists(env):
|
|
return True
|
|
|
|
|
|
def generate(env):
|
|
env.AddMethod(setup_task_limiter, 'SetupTaskLimiter')
|