From ff6326e5ab795e5e3ca94a12dd11806c6f875dd1 Mon Sep 17 00:00:00 2001 From: Charlie Swanson Date: Tue, 1 Sep 2015 12:58:04 -0400 Subject: [PATCH] SERVER-18273 Compute ranges of ports for each job in resmoke.py --- buildscripts/resmokelib/config.py | 5 + buildscripts/resmokelib/core/network.py | 127 +++++++++++++----- buildscripts/resmokelib/errors.py | 9 ++ buildscripts/resmokelib/parser.py | 7 + .../testing/fixtures/shardedcluster.py | 3 +- .../resmokelib/testing/fixtures/standalone.py | 3 +- buildscripts/resmokelib/testing/testcases.py | 6 + 7 files changed, 126 insertions(+), 34 deletions(-) diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index 7ce7d7cc9a6..16b9b79de14 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -33,6 +33,7 @@ MONGO_RUNNER_SUBDIR = "mongorunner" # Names below correspond to how they are specified via the command line or in the options YAML file. DEFAULTS = { + "basePort": 20000, "buildloggerUrl": "https://logkeeper.mongodb.org", "continueOnFailure": False, "dbpathPrefix": None, @@ -63,6 +64,10 @@ DEFAULTS = { # Variables that are set by the user at the command line or with --options. ## +# The starting port number to use for mongod and mongos processes spawned by resmoke.py and the +# mongo shell. +BASE_PORT = None + # The root url of the buildlogger server. BUILDLOGGER_URL = None diff --git a/buildscripts/resmokelib/core/network.py b/buildscripts/resmokelib/core/network.py index ec103df3dfc..b0e63a2acd7 100644 --- a/buildscripts/resmokelib/core/network.py +++ b/buildscripts/resmokelib/core/network.py @@ -1,47 +1,114 @@ """ -Helper to reserve a network port. +Class used to allocate ports for use by various mongod and mongos +processes involved in running the tests. """ from __future__ import absolute_import -import socket as _socket +import collections +import functools import threading +from .. import config +from .. import errors -class UnusedPort(object): + +def _check_port(func): """ - Acquires an unused port from the OS. + A decorator that verifies the port returned by the wrapped function + is in the valid range. + + Returns the port if it is valid, and raises a PortAllocationError + otherwise. """ - # Use a set to keep track of ports that are acquired from the OS to avoid returning duplicates. - # We do not remove ports from this set because they are used throughout the lifetime of - # resmoke.py for started mongod/mongos processes. - _ALLOCATED_PORTS = set() - _ALLOCATED_PORTS_LOCK = threading.Lock() + @functools.wraps(func) + def wrapper(*args, **kwargs): + port = func(*args, **kwargs) - def __init__(self): - self.num = None - self.__socket = None + if port < 0: + raise errors.PortAllocationError("Attempted to use a negative port") - def __enter__(self): - while True: - socket = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM) - socket.setsockopt(_socket.SOL_SOCKET, _socket.SO_REUSEADDR, 1) - socket.bind(("0.0.0.0", 0)) + if port > PortAllocator.MAX_PORT: + raise errors.PortAllocationError("Exhausted all available ports. Consider decreasing" + " the number of jobs, or using a lower base port") - port = socket.getsockname()[1] - with UnusedPort._ALLOCATED_PORTS_LOCK: - # Check whether the OS has already given us 'port'. - if port in UnusedPort._ALLOCATED_PORTS: - socket.close() - continue + return port - UnusedPort._ALLOCATED_PORTS.add(port) + return wrapper - self.num = port - self.__socket = socket - return self - def __exit__(self, *exc_info): - if self.__socket is not None: - self.__socket.close() +class PortAllocator(object): + """ + This class is responsible for allocating ranges of ports. + + It reserves a range of ports for each job with the first part of + that range used for the fixture started by that job, and the second + part of the range used for mongod and mongos processes started by + tests run by that job. + """ + + # A PortAllocator will not return any port greater than this number. + MAX_PORT = 2 ** 16 - 1 + + # Each job gets a contiguous range of _PORTS_PER_JOB ports, with job 0 getting the first block + # of ports, job 1 getting the second block, and so on. + _PORTS_PER_JOB = 30 + + # The first _PORTS_PER_FIXTURE ports of each range are reserved for the fixtures, the remainder + # of the port range is used by tests. + _PORTS_PER_FIXTURE = 10 + + _NUM_USED_PORTS_LOCK = threading.Lock() + + # Used to keep track of how many ports a fixture has allocated. + _NUM_USED_PORTS = collections.defaultdict(int) + + @classmethod + @_check_port + def next_fixture_port(cls, job_num): + """ + Returns the next port for a fixture to use. + + Raises a PortAllocationError if the fixture has requested more + ports than are reserved per job, or if the next port is not a + valid port number. + """ + with cls._NUM_USED_PORTS_LOCK: + start_port = config.BASE_PORT + (job_num * cls._PORTS_PER_JOB) + num_used_ports = cls._NUM_USED_PORTS[job_num] + next_port = start_port + num_used_ports + + cls._NUM_USED_PORTS[job_num] += 1 + + if next_port >= start_port + cls._PORTS_PER_FIXTURE: + raise errors.PortAllocationError( + "Fixture has requested more than the %d ports reserved per fixture" + % cls._PORTS_PER_FIXTURE) + + return next_port + + @classmethod + @_check_port + def min_test_port(cls, job_num): + """ + For the given job, returns the lowest port that is reserved for + use by tests. + + Raises a PortAllocationError if that port is higher than the + maximum port. + """ + return config.BASE_PORT + (job_num * cls._PORTS_PER_JOB) + cls._PORTS_PER_FIXTURE + + @classmethod + @_check_port + def max_test_port(cls, job_num): + """ + For the given job, returns the highest port that is reserved + for use by tests. + + Raises a PortAllocationError if that port is higher than the + maximum port. + """ + next_range_start = config.BASE_PORT + ((job_num + 1) * cls._PORTS_PER_JOB) + return next_range_start - 1 diff --git a/buildscripts/resmokelib/errors.py b/buildscripts/resmokelib/errors.py index 8243b4ce157..d07dd2078e6 100644 --- a/buildscripts/resmokelib/errors.py +++ b/buildscripts/resmokelib/errors.py @@ -33,3 +33,12 @@ class ServerFailure(TestFailure): as a failure. """ pass + + +class PortAllocationError(ResmokeError): + """ + Exception that is raised by the PortAllocator if a port is requested + outside of the range of valid ports, or if a fixture requests more + ports than were reserved for that job. + """ + pass diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py index 3c6f453f34a..faf9b76eb9c 100644 --- a/buildscripts/resmokelib/parser.py +++ b/buildscripts/resmokelib/parser.py @@ -17,6 +17,7 @@ from .. import resmokeconfig # Mapping of the attribute of the parsed arguments (dest) to its key as it appears in the options # YAML configuration file. Most should only be converting from snake_case to camelCase. DEST_TO_CONFIG = { + "base_port": "basePort", "buildlogger_url": "buildloggerUrl", "continue_on_failure": "continueOnFailure", "dbpath_prefix": "dbpathPrefix", @@ -71,6 +72,11 @@ def parse_command_line(): parser.add_option("--options", dest="options_file", metavar="OPTIONS", help="A YAML file that specifies global options to resmoke.py.") + parser.add_option("--basePort", dest="base_port", metavar="PORT", + help=("The starting port number to use for mongod and mongos processes" + " spawned by resmoke.py or the tests themselves. Each fixture and Job" + " allocates a contiguous range of ports.")) + parser.add_option("--buildloggerUrl", action="store", dest="buildlogger_url", metavar="URL", help="The root url of the buildlogger server.") @@ -186,6 +192,7 @@ def update_config_vars(values): if values[dest] is not None: config[config_var] = values[dest] + _config.BASE_PORT = int(config.pop("basePort")) _config.BUILDLOGGER_URL = config.pop("buildloggerUrl") _config.DBPATH_PREFIX = _expand_user(config.pop("dbpathPrefix")) _config.DBTEST_EXECUTABLE = _expand_user(config.pop("dbtest")) diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py index 9551d93c05d..92659e3338f 100644 --- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py +++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py @@ -266,8 +266,7 @@ class _MongoSFixture(interface.Fixture): self.mongos_options["chunkSize"] = 50 if "port" not in self.mongos_options: - with core.network.UnusedPort() as port: - self.mongos_options["port"] = port.num + self.mongos_options["port"] = core.network.PortAllocator.next_fixture_port(self.job_num) self.port = self.mongos_options["port"] mongos = core.programs.mongos_program(self.logger, diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py index ff82356d020..9b867f8a0ce 100644 --- a/buildscripts/resmokelib/testing/fixtures/standalone.py +++ b/buildscripts/resmokelib/testing/fixtures/standalone.py @@ -69,8 +69,7 @@ class MongoDFixture(interface.Fixture): pass if "port" not in self.mongod_options: - with core.network.UnusedPort() as port: - self.mongod_options["port"] = port.num + self.mongod_options["port"] = core.network.PortAllocator.next_fixture_port(self.job_num) self.port = self.mongod_options["port"] mongod = core.programs.mongod_program(self.logger, diff --git a/buildscripts/resmokelib/testing/testcases.py b/buildscripts/resmokelib/testing/testcases.py index cef1f11ef66..e6f2ff94dc9 100644 --- a/buildscripts/resmokelib/testing/testcases.py +++ b/buildscripts/resmokelib/testing/testcases.py @@ -307,6 +307,12 @@ class JSTestCase(TestCase): global_vars["MongoRunner.dataDir"] = data_dir global_vars["MongoRunner.dataPath"] = data_path + + min_port = core.network.PortAllocator.min_test_port(fixture.job_num) + max_port = core.network.PortAllocator.max_test_port(fixture.job_num) + global_vars["MongoRunner.minPort"] = min_port + global_vars["MongoRunner.maxPort"] = max_port + self.shell_options["global_vars"] = global_vars shutil.rmtree(data_dir, ignore_errors=True)