diff --git a/.gitignore b/.gitignore index 1683b9a9764..450a168ee34 100644 --- a/.gitignore +++ b/.gitignore @@ -282,3 +282,12 @@ bazelisk # generated configs for antithesis antithesis/antithesis_config + +# artifacts from antithesis docker base image builds +antithesis-dist-test +buildscripts/antithesis/base_images/mongo_binaries/dist-test +buildscripts/antithesis/base_images/mongo_binaries/libvoidstar.so + +buildscripts/antithesis/base_images/workload/src +buildscripts/antithesis/base_images/workload/mongo +buildscripts/antithesis/base_images/workload/libvoidstar.so diff --git a/antithesis/templates/docker_compose_template.yml.jinja b/antithesis/templates/docker_compose_template.yml.jinja index 19fd78ec3fb..e4c6363d0ba 100644 --- a/antithesis/templates/docker_compose_template.yml.jinja +++ b/antithesis/templates/docker_compose_template.yml.jinja @@ -5,7 +5,7 @@ services: configsvr{{ i }}: container_name: configsvr{{ i }} hostname: configsvr{{ i }} - image: mongo-binaries:evergreen-latest-master + image: mongo-binaries:{{ tag }} volumes: - ./logs/configsvr{{ i }}:/var/log/mongodb/ - ./scripts:/scripts/ @@ -22,7 +22,7 @@ services: mongod{{ i }}: container_name: mongod{{ i }} hostname: mongod{{ i }} - image: mongo-binaries:evergreen-latest-master + image: mongo-binaries:{{ tag }} volumes: - ./logs/mongod{{ i }}:/var/log/mongodb/ - ./scripts:/scripts/ @@ -38,7 +38,7 @@ services: mongos{{ m }}: container_name: mongos{{ m }} hostname: mongos{{ m }} - image: mongo-binaries:evergreen-latest-master + image: mongo-binaries:{{ tag }} volumes: - ./logs/mongos{{ m }}:/var/log/mongodb/ - ./scripts:/scripts/ @@ -60,7 +60,7 @@ services: workload: container_name: workload hostname: workload - image: workload:evergreen-latest-master + image: workload:{{ tag }} volumes: - ./logs/workload:/var/log/resmoke/ - ./scripts:/scripts/ diff --git a/antithesis/templates/run_suite_template.sh.jinja b/antithesis/templates/run_suite_template.sh.jinja new file mode 100644 index 00000000000..524720ed70f --- /dev/null +++ b/antithesis/templates/run_suite_template.sh.jinja @@ -0,0 +1,10 @@ +# Script to run the target antithesis suite + +final_exit_code=0 +sudo docker-compose down +sudo docker-compose up -d +sudo docker exec workload /bin/bash -c \ + "cd resmoke && . python3-venv/bin/activate && python3 buildscripts/resmoke.py run --suite {{ suite }} --sanityCheck" +final_exit_code=$(echo $?) +sudo docker-compose down +exit $final_exit_code diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index 368a1a5df36..444f3b54a8d 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -160,6 +160,9 @@ DEFAULTS = { # Limit the number of tests to execute "max_test_queue_size": None, + + # Sanity check + "sanity_check": False, } _SuiteOptions = collections.namedtuple("_SuiteOptions", [ @@ -611,3 +614,6 @@ EXPANSIONS_FILE = "../expansions.yml" if 'CI' in os.environ else None # Symbolizer secrets SYMBOLIZER_CLIENT_SECRET = None SYMBOLIZER_CLIENT_ID = None + +# Sanity check +SANITY_CHECK = False diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py index bf613391ac4..14046936281 100644 --- a/buildscripts/resmokelib/configure_resmoke.py +++ b/buildscripts/resmokelib/configure_resmoke.py @@ -359,6 +359,7 @@ or explicitly pass --installDir to the run subcommand of buildscripts/resmoke.py _config.TAG_FILES = config.pop("tag_files") _config.TRANSPORT_LAYER = config.pop("transport_layer") _config.USER_FRIENDLY_OUTPUT = config.pop("user_friendly_output") + _config.SANITY_CHECK = config.pop("sanity_check") # Internal testing options. _config.INTERNAL_PARAMS = config.pop("internal_params") diff --git a/buildscripts/resmokelib/errors.py b/buildscripts/resmokelib/errors.py index d95cb4ef791..08736170eb7 100644 --- a/buildscripts/resmokelib/errors.py +++ b/buildscripts/resmokelib/errors.py @@ -92,3 +92,9 @@ class TagFileDoesNotExistError(ResmokeError): """Exception raised when a tag file is passed into resmoke that does not exist.""" pass + + +class RequiresForceRemove(ResmokeError): + """Exception raised when a directory cannot be removed.""" + + pass diff --git a/buildscripts/resmokelib/generate_docker_compose/__init__.py b/buildscripts/resmokelib/generate_docker_compose/__init__.py index 1b11581a7a4..e9d929d6083 100644 --- a/buildscripts/resmokelib/generate_docker_compose/__init__.py +++ b/buildscripts/resmokelib/generate_docker_compose/__init__.py @@ -1,11 +1,13 @@ """Generate a docker compose configuration and all necessary infrastructure.""" +import os +import sys +from buildscripts.resmokelib.errors import InvalidMatrixSuiteError, RequiresForceRemove from buildscripts.resmokelib.plugin import PluginInterface, Subcommand -from buildscripts.resmokelib.testing.docker_cluster_config_writer import DockerClusterConfigWriter -from buildscripts.resmokelib.testing.fixtures import _builder +from buildscripts.resmokelib.testing.docker_cluster_image_builder import DockerComposeImageBuilder _HELP = """ -Generate a docker compose configuration and all necessary infrastructure. +Generate a docker compose configuration and all necessary infrastructure -- including base images. """ _COMMAND = "generate-docker-compose" @@ -13,10 +15,19 @@ _COMMAND = "generate-docker-compose" class GenerateDockerCompose(Subcommand): """Generate docker compose configuration and infrastructure.""" - def __init__(self, suite_name): - """Constructor.""" - self._fixture = _builder.make_dummy_fixture(suite_name) - self._suite_name = suite_name + def __init__(self, antithesis_suite_name, build_base_images, tag, in_evergreen): + """ + Constructor for GenerateDockerCompose subcommand. + + :param antithesis_suite_name: The antithesis suite to generate a docker compose configuration for. + :param build_base_images: Whether to build the base images or not. + :param tag: The tag to use for the docker images and docker-compose file. + :param in_evergreen: Whether this is running in Evergreen or not. + """ + self._antithesis_suite_name = antithesis_suite_name + self._build_base_images = build_base_images + self._tag = tag + self._in_evergreen = in_evergreen def execute(self) -> None: """ @@ -24,12 +35,33 @@ class GenerateDockerCompose(Subcommand): :return: None """ - if self._fixture.__class__.__name__ == "ShardedClusterFixture": - DockerClusterConfigWriter(self._suite_name, - self._fixture).generate_docker_sharded_cluster_config() - else: - print("Generating docker compose infrastructure for this fixture is not yet supported.") - exit(1) + try: + image_builder = DockerComposeImageBuilder(self._tag, self._in_evergreen) + if self._antithesis_suite_name: + image_builder.build_config_image(self._antithesis_suite_name) + if self._build_base_images: + image_builder.build_base_images() + if self._build_base_images and self._antithesis_suite_name: + success_message = f""" + Successfully generated docker compose configuration and built required base images. + + You can run the following command to verify that this docker compose configuration works: + `cd antithesis/antithesis_config/{self._antithesis_suite_name} && bash run_suite.sh` + """ + print(success_message) + + except RequiresForceRemove as exc: + print(exc) + sys.exit(2) + except AssertionError as exc: + print(exc) + sys.exit(3) + except InvalidMatrixSuiteError as exc: + print(exc) + sys.exit(4) + except Exception as exc: + raise Exception( + "Something unexpected happened while building antithesis images.") from exc class GenerateDockerComposePlugin(PluginInterface): @@ -43,10 +75,20 @@ class GenerateDockerComposePlugin(PluginInterface): :return: None """ parser = subparsers.add_parser(_COMMAND, help=_HELP) + parser.add_argument("-t", "--tag", dest="tag", metavar="TAG", default="local-development", + help="Build base images needed for the docker compose configuration.") + parser.add_argument("-s", "--skip-base-image-build", dest="skip_base_image_build", + default=False, action="store_true", + help="Skip building images for the docker compose configuration.") parser.add_argument( - "--suite", dest="suite", metavar="SUITE", - help=("Suite file from the resmokeconfig/suites/ directory." - " Use the basename without the .yml extension.")) + "--in-evergreen", dest="in_evergreen", default=False, action="store_true", + help="If this is running in Evergreen, certain artifacts are expected to already exist." + ) + parser.add_argument( + nargs="?", dest="antithesis_suite", metavar="SUITE", help= + ("Antithesis Matrix Suite file from the resmokeconfig/matrix_suites/mappings directory." + " Use the basename without the .yml extension. If empty, only base images will be built." + )) def parse(self, subcommand, parser, parsed_args, **kwargs): """ @@ -58,8 +100,10 @@ class GenerateDockerComposePlugin(PluginInterface): :param kwargs: additional args :return: None or a Subcommand """ - if subcommand != _COMMAND: return None - return GenerateDockerCompose(parsed_args.suite) + build_base_images = parsed_args.skip_base_image_build is False + + return GenerateDockerCompose(parsed_args.antithesis_suite, build_base_images, + parsed_args.tag, parsed_args.in_evergreen) diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py index c879a03e23b..1ad2381fb9c 100644 --- a/buildscripts/resmokelib/run/__init__.py +++ b/buildscripts/resmokelib/run/__init__.py @@ -791,6 +791,11 @@ class RunPlugin(PluginInterface): " only tests which have at least one of the specified tags will be" " run.")) + parser.add_argument( + "--sanityCheck", action="store_true", dest="sanity_check", help= + "Truncate the test queue to 1 item, just in order to verify the suite is properly set up." + ) + parser.add_argument( "--includeWithAllTags", action="append", dest="include_with_all_tags", metavar="TAG1,TAG2", diff --git a/buildscripts/resmokelib/testing/docker_cluster_config_writer.py b/buildscripts/resmokelib/testing/docker_cluster_config_writer.py index 9720c6aa1ad..7724c68212d 100644 --- a/buildscripts/resmokelib/testing/docker_cluster_config_writer.py +++ b/buildscripts/resmokelib/testing/docker_cluster_config_writer.py @@ -5,28 +5,61 @@ import os import re import shutil -from buildscripts.resmokelib import config -from buildscripts.resmokelib.core import programs from jinja2 import Environment, FileSystemLoader +from buildscripts.resmokelib.core import programs +from buildscripts.resmokelib.errors import InvalidMatrixSuiteError, RequiresForceRemove +from buildscripts.resmokelib.suitesconfig import get_suite +from buildscripts.resmokelib.testing.fixtures import _builder + MONGOS_PORT = 27017 MONGOD_PORT = 27018 CONFIG_PORT = 27019 +def get_antithesis_base_suite_fixture(antithesis_suite_name) -> None: + """ + Get the base suite fixture to use for generating docker compose configuration. + + :param antithesis_suite_name: The antithesis suite to find the base suite fixture for. + """ + antithesis_suite = get_suite(antithesis_suite_name) + if not antithesis_suite.is_matrix_suite() or not antithesis_suite.is_antithesis_suite(): + raise InvalidMatrixSuiteError( + f"The specified suite is not an antithesis matrix suite: {antithesis_suite.get_name()}") + + antithesis_fixture = antithesis_suite.get_executor_config()["fixture"]["class"] + if antithesis_fixture != "ExternalShardedClusterFixture": + raise InvalidMatrixSuiteError( + "Generating docker compose infrastructure for this external fixture is not yet supported" + ) + + antithesis_base_suite = antithesis_suite.get_executor_config()["fixture"]["original_suite_name"] + base_suite_fixture = _builder.make_dummy_fixture(antithesis_base_suite) + if base_suite_fixture.__class__.__name__ != "ShardedClusterFixture": + raise InvalidMatrixSuiteError( + "Generating docker compose infrastructure for this base suite fixture is not yet supported.{}" + ) + + return base_suite_fixture + + class DockerClusterConfigWriter(object): """Create necessary files and topology for a suite to run with Antithesis.""" - def __init__(self, suite_name, fixture): + def __init__(self, antithesis_suite_name, tag): """ Initialize the class with the specified fixture. - :param suite: Suite we wish to generate files and topology for. - :param fixture: Fixture for the suite we wish to generate files and topology for. + :param antithesis_suite_name: Suite we wish to generate files and topology for. + :param tag: Tag to use for the docker compose configuration and/or base images. """ self.ip_address = 1 - self.fixture = fixture - self.suite_path = os.path.join(os.getcwd(), f"antithesis/antithesis_config/{suite_name}") + self.antithesis_suite_name = antithesis_suite_name + self.fixture = get_antithesis_base_suite_fixture(antithesis_suite_name) + self.tag = tag + self.build_context = os.path.join(os.getcwd(), + f"antithesis/antithesis_config/{antithesis_suite_name}") self.jinja_env = Environment( loader=FileSystemLoader(os.path.join(os.getcwd(), "antithesis/templates/"))) @@ -36,8 +69,8 @@ class DockerClusterConfigWriter(object): :return: None. """ - # Create directory structure - self.create_directory_structure() + # Create volume directory structure + self.create_volume_directories() # Create configsvr init scripts self.create_configsvr_init() # Create mongod init scripts @@ -50,6 +83,8 @@ class DockerClusterConfigWriter(object): self.write_docker_compose() # Create dockerfile self.write_dockerfile() + # Create run suite script + self.write_run_suite_script() def write_docker_compose(self): """ @@ -57,13 +92,13 @@ class DockerClusterConfigWriter(object): :return: None. """ - with open(os.path.join(self.suite_path, "docker-compose.yml"), 'w') as file: + with open(os.path.join(self.build_context, "docker-compose.yml"), 'w') as file: template = self.jinja_env.get_template("docker_compose_template.yml.jinja") file.write( template.render( - num_configsvr=self.fixture.configsvr_options.get( - "num_nodes", 1), num_shard=self.fixture.num_shards, num_node_per_shard=self. - fixture.num_rs_nodes_per_shard, num_mongos=self.fixture.num_mongos, + num_configsvr=self.fixture.configsvr_options.get("num_nodes", 1), + num_shard=self.fixture.num_shards, num_node_per_shard=self.fixture. + num_rs_nodes_per_shard, num_mongos=self.fixture.num_mongos, tag=self.tag, get_and_increment_ip_address=self.get_and_increment_ip_address) + "\n") def write_workload_init(self): @@ -72,7 +107,7 @@ class DockerClusterConfigWriter(object): :return: None. """ - with open(os.path.join(self.suite_path, "scripts/workload_init.py"), 'w') as file: + with open(os.path.join(self.build_context, "scripts/workload_init.py"), 'w') as file: template = self.jinja_env.get_template("workload_init_template.py.jinja") file.write(template.render() + "\n") @@ -82,26 +117,36 @@ class DockerClusterConfigWriter(object): :return: None. """ - with open(os.path.join(self.suite_path, "Dockerfile"), 'w') as file: + with open(os.path.join(self.build_context, "Dockerfile"), 'w') as file: template = self.jinja_env.get_template("dockerfile_template.jinja") file.write(template.render() + "\n") - def create_directory_structure(self): + def create_volume_directories(self): """ - Create the necessary directories for Docker topology. These will overwrite any existing topology for this suite. + Create the necessary volume directories for the Docker topology. :return: None. """ paths = [ - self.suite_path, - os.path.join(self.suite_path, "scripts"), - os.path.join(self.suite_path, "logs"), - os.path.join(self.suite_path, "data"), - os.path.join(self.suite_path, "debug") + self.build_context, + os.path.join(self.build_context, "scripts"), + os.path.join(self.build_context, "logs"), + os.path.join(self.build_context, "data"), + os.path.join(self.build_context, "debug") ] for p in paths: if os.path.exists(p): - shutil.rmtree(p) + try: + shutil.rmtree(p) + except Exception as exc: + exception_text = f""" + Could not remove directory due to old artifacts from a previous run. + + Please remove this directory and try again -- you may need to force remove: + `{os.path.relpath(p)}` + """ + raise RequiresForceRemove(exception_text) from exc + os.makedirs(p) def get_and_increment_ip_address(self): @@ -111,7 +156,7 @@ class DockerClusterConfigWriter(object): :return: ip_address. """ if self.ip_address > 24: - self._resmoke_logger.info("ipv4_address exceeded 10.20.20.24. Exiting with code: 2") + print(f"Exiting with code 2 -- ipv4_address exceeded 10.20.20.24: {self.ip_address}") sys.exit(2) self.ip_address += 1 return self.ip_address @@ -147,7 +192,7 @@ class DockerClusterConfigWriter(object): :param args: List of arguments for initiating the current node. :return: None. """ - script_path = os.path.join(self.suite_path, f"scripts/{node_name}_init.sh") + script_path = os.path.join(self.build_context, f"scripts/{node_name}_init.sh") with open(script_path, 'w') as file: template = self.jinja_env.get_template("node_init_template.sh.jinja") file.write(template.render(command=' '.join(args)) + "\n") @@ -210,7 +255,7 @@ class DockerClusterConfigWriter(object): :param args: List of arguments that need to be set for mongod init. :return: None. """ - with open(os.path.join(self.suite_path, f"scripts/{mongos_name}_init.py"), 'w') as file: + with open(os.path.join(self.build_context, f"scripts/{mongos_name}_init.py"), 'w') as file: template = self.jinja_env.get_template("mongos_init_template.py.jinja") file.write( template.render(mongos_name=mongos_name, configsvr=self.fixture.configsvr, @@ -262,3 +307,15 @@ class DockerClusterConfigWriter(object): "electionTimeoutMillis": 2000, "heartbeatTimeoutSecs": 1, "chainingAllowed": False }) return settings + + def write_run_suite_script(self): + """ + Write the `run_suite.sh` file which starts up the docker cluster and runs a sanity check. + + This ensures that the configuration for the suite works as expected. + + :return: None. + """ + with open(os.path.join(self.build_context, "run_suite.sh"), 'w') as file: + template = self.jinja_env.get_template("run_suite_template.sh.jinja") + file.write(template.render(suite=self.antithesis_suite_name) + "\n") diff --git a/buildscripts/resmokelib/testing/docker_cluster_image_builder.py b/buildscripts/resmokelib/testing/docker_cluster_image_builder.py new file mode 100644 index 00000000000..9b4c16e3ed4 --- /dev/null +++ b/buildscripts/resmokelib/testing/docker_cluster_image_builder.py @@ -0,0 +1,277 @@ +import os +import shutil +import subprocess +import sys + +import git + +from buildscripts.resmokelib.testing.docker_cluster_config_writer import DockerClusterConfigWriter + + +class DockerComposeImageBuilder: + """Build images needed to run a resmoke suite against a MongoDB Docker Container topology.""" + + def __init__(self, tag, in_evergreen): + """ + Constructs a `DockerComposeImageBuilder` which can build images locally and in CI. + + :param tag: The tag to use for these images. + :param in_evergreen: Whether this is running in Evergreen or not. + """ + self.tag = tag + self.in_evergreen = in_evergreen + + # Build context constants + self.WORKLOAD_BUILD_CONTEXT = "buildscripts/antithesis/base_images/workload" + self.WORKLOAD_DOCKERFILE = f"{self.WORKLOAD_BUILD_CONTEXT}/Dockerfile" + + self.MONGO_BINARIES_BUILD_CONTEXT = "buildscripts/antithesis/base_images/mongo_binaries" + self.MONGO_BINARIES_DOCKERFILE = f"{self.MONGO_BINARIES_BUILD_CONTEXT}/Dockerfile" + + # Artifact constants + self.MONGODB_BINARIES_RELATIVE_DIR = "dist-test" if in_evergreen else "antithesis-dist-test" + self.MONGO_BINARY = f"{self.MONGODB_BINARIES_RELATIVE_DIR}/bin/mongo" + self.MONGOD_BINARY = f"{self.MONGODB_BINARIES_RELATIVE_DIR}/bin/mongod" + self.MONGOS_BINARY = f"{self.MONGODB_BINARIES_RELATIVE_DIR}/bin/mongos" + + self.LIBVOIDSTAR_PATH = "/usr/lib/libvoidstar.so" + self.MONGODB_DEBUGSYMBOLS = "mongo-debugsymbols.tgz" + + def build_base_images(self): + """ + Build the base images needed for the docker configuration. + + :return: None. + """ + self._fetch_mongodb_binaries() + print("Building base images...") + self._build_mongo_binaries_image() + self._build_workload_image() + print("Done building base images.") + + def build_config_image(self, antithesis_suite_name): + """ + Build the antithesis config image containing the `docker-compose.yml` file and volumes for the suite. + + :param antithesis_suite_name: The antithesis suite to build a docker compose config image for. + :param tag: Tag to use for the docker compose configuration and/or base images. + """ + + # Build out the directory structure and write the startup scripts for the config image at runtime + print(f"Prepping antithesis config image build context for `{antithesis_suite_name}`...") + config_image_writer = DockerClusterConfigWriter(antithesis_suite_name, self.tag) + config_image_writer.generate_docker_sharded_cluster_config() + + # Our official builds happen in Evergreen. Assert debug symbols are on system. + # If this is running locally, this is for development purposes only and debug symbols are not required. + if self.in_evergreen: + assert os.path.exists(self.MONGODB_DEBUGSYMBOLS + ), f"No debug symbols available at: {self.MONGODB_DEBUGSYMBOLS}" + print("Running in Evergreen -- copying debug symbols to build context...") + shutil.copy(self.MONGODB_DEBUGSYMBOLS, + os.path.join(config_image_writer.build_context, "debug")) + + print( + f"Done setting up antithesis config image build context for `{antithesis_suite_name}..." + ) + print("Building antithesis config image...") + subprocess.run([ + "docker", "build", "-t", f"{antithesis_suite_name}:{self.tag}", "-f", + f"{config_image_writer.build_context}/Dockerfile", config_image_writer.build_context + ], stdout=sys.stdout, stderr=sys.stderr, check=True) + print("Done building antithesis config image.") + + def _build_workload_image(self): + """ + Build the workload image. + + :param tag: Tag to use for the image. + :return: None. + """ + + print("Prepping `workload` image build context...") + # Set up build context + self._copy_mongo_binary_to_build_context(self.WORKLOAD_BUILD_CONTEXT) + self._clone_mongo_repo_to_build_context(self.WORKLOAD_BUILD_CONTEXT) + self._add_libvoidstar_to_build_context(self.WORKLOAD_BUILD_CONTEXT) + + # Build docker image + print("Building workload image...") + subprocess.run([ + "docker", "build", "-t", f"workload:{self.tag}", "-f", self.WORKLOAD_DOCKERFILE, + self.WORKLOAD_BUILD_CONTEXT + ], stdout=sys.stdout, stderr=sys.stderr, check=True) + print("Done building workload image.") + + def _build_mongo_binaries_image(self): + """ + Build the mongo-binaries image. + + :return: None. + """ + + print("Prepping `mongo binaries` image build context...") + # Set up build context + self._copy_mongodb_binaries_to_build_context(self.MONGO_BINARIES_BUILD_CONTEXT) + self._add_libvoidstar_to_build_context(self.MONGO_BINARIES_BUILD_CONTEXT) + + # Build docker image + print("Building mongo binaries image...") + subprocess.run([ + "docker", "build", "-t", f"mongo-binaries:{self.tag}", "-f", + self.MONGO_BINARIES_DOCKERFILE, self.MONGO_BINARIES_BUILD_CONTEXT + ], stdout=sys.stdout, stderr=sys.stderr, check=True) + print("Done building mongo binaries image.") + + def _fetch_mongodb_binaries(self): + """ + Get MongoDB binaries -- if running locally -- and verify existence/validity. + + In CI the binaries should already exist. In that case, we want to verify binary existence and + that they are linked with `libvoidstar`. + + :return: None. + """ + mongodb_binaries_destination = os.path.join(self.MONGODB_BINARIES_RELATIVE_DIR, "bin") + + # If local, fetch the binaries. + if not self.in_evergreen: + # Clean up any old artifacts in the build context. + if os.path.exists(mongodb_binaries_destination): + print("Removing old MongoDB binaries...") + shutil.rmtree(mongodb_binaries_destination) + + # Ensure that `db-contrib-tool` is installed locally + db_contrib_tool_error = """ + Could not find `db-contrib-tool` installation locally. + + You can install `db-contrib-tool` with the following command: + `pip install db-contrib-tool` OR `pipx install db-contrib-tool` + + More info on `db-contrib-tool` available here: `https://github.com/10gen/db-contrib-tool` + """ + assert subprocess.run(["which", + "db-contrib-tool"]).returncode == 0, db_contrib_tool_error + + # Use `db-contrib-tool` to get MongoDB binaries for this image + print("Fetching MongoDB binaries for image build...") + subprocess.run([ + "db-contrib-tool", "setup-repro-env", "--variant", "ubuntu1804", "--linkDir", + mongodb_binaries_destination, "master" + ], stdout=sys.stdout, stderr=sys.stderr, check=True) + + # Verify the binaries were downloaded successfully + for required_binary in [self.MONGO_BINARY, self.MONGOD_BINARY, self.MONGOS_BINARY]: + assert os.path.exists( + required_binary + ), f"Could not find Ubuntu 18.04 MongoDB binary at: {required_binary}" + + # Our official builds happen in Evergreen. + # We want to ensure the binaries are linked with `libvoidstar.so` during image build. + if self.in_evergreen: + assert "libvoidstar" in subprocess.run( + ["ldd", required_binary], + check=True, + capture_output=True, + ).stdout.decode( + "utf-8"), f"MongoDB binary is not linked to `libvoidstar.so`: {required_binary}" + + def _copy_mongo_binary_to_build_context(self, dir_path): + """ + Copy the mongo binary to the build context. + + :param dir_path: Directory path to add mongo binary to. + """ + mongo_binary_destination = os.path.join(dir_path, "mongo") + + print("Copy mongo binary to build context...") + # Clean up any old artifacts in the build context + if os.path.exists(mongo_binary_destination): + print("Cleaning up mongo binary from build context first...") + os.remove(mongo_binary_destination) + shutil.copy(self.MONGO_BINARY, mongo_binary_destination) + print("Done copying mongo binary to build context.") + + def _clone_mongo_repo_to_build_context(self, dir_path): + """ + Clone the mongo repo to the build context. + + :param dir_path: Directory path to clone mongo repo to. + """ + mongo_repo_destination = os.path.join(dir_path, "src") + + # Our official builds happen in Evergreen. Assert there's already a `mongo` repo in the build context. + # This is because we cannot rely on the standard "git clone" command to include uncommitted changes applied from a `patch`. + # Instead, we rely on Evergreen's `git.get_project` which will correctly clone the repo and apply changes from the `patch`. + if self.in_evergreen: + assert os.path.exists( + mongo_repo_destination), f"No `mongo` repo available at: {mongo_repo_destination}" + print("Running in Evergreen -- no need to clone `mongo` repo since it already exists.") + return + + # Clean up any old artifacts in the build context. + if os.path.exists(mongo_repo_destination): + print("Cleaning up MongoDB repo from build context first...") + shutil.rmtree(mongo_repo_destination) + + print("Cloning current MongoDB repo to build context...") + clone_repo_warning_message = """ + - If you have uncommitted changes, they will not be included in the `workload` image. + - If you would like to include these changes, commit your changes and try again. + """ + print(clone_repo_warning_message) + + # Copy the mongo repo to the build context. + # If this fails to clone, the `git` library will raise an exception. + git.Repo("./").clone(mongo_repo_destination) + print("Done cloning MongoDB repo to build context.") + + def _copy_mongodb_binaries_to_build_context(self, dir_path): + """ + Copy the MongodDB binaries to the build context. + + :param dir_path: Directory path to add MongoDB binaries to. + """ + print("Copy all MongoDB binaries to build context...") + mongodb_binaries_destination = os.path.join(dir_path, "dist-test") + + # Clean up any old artifacts in the build context + if os.path.exists(mongodb_binaries_destination): + print("Cleaning up all MongoDB binaries from build context first...") + shutil.rmtree(mongodb_binaries_destination) + shutil.copytree(self.MONGODB_BINARIES_RELATIVE_DIR, mongodb_binaries_destination) + print("Done copying all MongoDB binaries to build context.") + + def _add_libvoidstar_to_build_context(self, dir_path): + """ + Add the antithesis instrumentation library from the system to the build context. + + :param dir_path: Directory path to add `libvoidstar.so` to. + """ + print("Add `livoidstar.so` to build context...") + libvoidstar_destination = os.path.join(dir_path, "libvoidstar.so") + + # Clean up any old artifacts in the build context + if os.path.exists(libvoidstar_destination): + print("Cleaning up `libvoidstar.so` from build context first...") + os.remove(libvoidstar_destination) + + # Our official builds happen in Evergreen. Assert a "real" `libvoidstar.so` is on system. + if self.in_evergreen: + assert os.path.exists( + self.LIBVOIDSTAR_PATH), f"No `libvoidstar.so` available at: {self.LIBVOIDSTAR_PATH}" + print("Running in Evergreen -- using system `libvoidstar.so`.") + shutil.copy(self.LIBVOIDSTAR_PATH, libvoidstar_destination) + print("Done copying `libvoidstar.so` from system to build context") + + else: + disclaimer_message = """ + This is a stub `libvoidstar.so` file. It does not actually do anything. For local development, we + don't expect developers to have `libvoidstar.so` available, but it is necessary for building the + base images and is part of the base image Dockerfile(s). + """ + with open(libvoidstar_destination, "w") as file: + file.write(disclaimer_message) + print( + "Done writing stub `libvoidstar.so` to build context -- This is for development only." + ) diff --git a/buildscripts/resmokelib/testing/executor.py b/buildscripts/resmokelib/testing/executor.py index bb5ffdb2e05..65f1c008ae7 100644 --- a/buildscripts/resmokelib/testing/executor.py +++ b/buildscripts/resmokelib/testing/executor.py @@ -312,6 +312,8 @@ class TestSuiteExecutor(object): for test_name in self._suite.make_test_case_names_list(): queue_elem = self._create_queue_elem_for_test_name(test_name) test_cases.append(queue_elem) + if _config.SANITY_CHECK: + break test_queue.add_test_cases(test_cases) return test_queue diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 411f9c33432..132c605c218 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -3119,7 +3119,7 @@ buildvariants: scons_cache_scope: shared tasks: - name: compile_and_archive_dist_test_TG - - name: antithesis_TG + - name: .antithesis - name: generate_buildid_to_debug_symbols_mapping - <<: *enterprise-windows-nopush-template diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index 197954d7d26..ea447f00d01 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -300,24 +300,15 @@ variables: teardown_group: - func: "umount shared scons directory" -- &antithesis_config_build_and_push_template - name: antithesis_config_build_and_push_template +- &antithesis_task_template + name: antithesis_task_template tags: ["antithesis"] depends_on: - - name: antithesis_base_image_build_and_push + name: archive_dist_test_debug commands: - - func: "antithesis config image build and push" - vars: - suite: suite_name - -- &antithesis_config_test_template - name: antithesis_config_test_template - tags: ["antithesis"] - exec_timeout_secs: 14400 # Time out the task if it runs for more than 4 hours. - depends_on: - - name: antithesis_config_build_and_push_task - commands: - - func: "antithesis config image test" + - func: "do setup" + - func: "do setup for antithesis" + - func: "antithesis image build and push" vars: suite: suite_name @@ -921,6 +912,23 @@ functions: - *collect_system_resource_info - *collect_ulimit_info + "do setup for antithesis": + - command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + remote_file: ${mongo_debugsymbols} + bucket: mciuploads + local_file: src/mongo-debugsymbols.${ext|tgz} + - command: subprocess.exec + params: + binary: bash + args: + - "./src/evergreen/modify_debug_symbols.sh" + - command: git.get_project + params: + directory: src/buildscripts/antithesis/base_images/workload/src + "do non-compile setup": - command: manifest.load - *git_get_project @@ -2469,34 +2477,13 @@ functions: content_type: text/plain display_name: multiversion_exclude_tags.yml from resmoke invocation - "antithesis config image build and push": - - *f_expansions_write - - command: s3.get - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - remote_file: ${mongo_debugsymbols} - bucket: mciuploads - local_file: src/mongo-debugsymbols.${ext|tgz} - - command: subprocess.exec - params: - binary: bash - args: - - "./src/evergreen/modify_debug_symbols.sh" + "antithesis image build and push": - *f_expansions_write - command: subprocess.exec params: binary: bash args: - - "./src/evergreen/antithesis_config_image_build_and_push.sh" - - "antithesis config image test": - - *f_expansions_write - - command: subprocess.exec - params: - binary: bash - args: - - "./src/evergreen/antithesis_config_image_test.sh" + - "./src/evergreen/antithesis_image_build_and_push.sh" # Pre task steps pre: @@ -2687,6 +2674,7 @@ tasks: - "jstests/**" - "patch_files.txt" - "evergreen/**" + - "antithesis/**" - "src/**.idl" - "src/mongo/client/sdam/json_tests/sdam_tests/**" - "src/mongo/client/sdam/json_tests/server_selection_tests/**" @@ -8565,31 +8553,6 @@ tasks: args: - "./src/evergreen/feature_flag_tags_check.sh" -- name: antithesis_base_image_build_and_push - tags: ["antithesis"] - depends_on: - - name: archive_dist_test_debug - commands: - - *f_expansions_write - - func: "git get project no modules" - - func: "f_expansions_write" - - func: "kill processes" - - func: "cleanup environment" - - func: "set up venv" - - func: "do setup" - - command: s3.get - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - remote_file: ${mongo_debugsymbols} - bucket: mciuploads - local_file: src/mongo-debugsymbols.${ext|tgz} - - command: subprocess.exec - params: - binary: bash - args: - - "./src/evergreen/antithesis_base_image_build_and_push.sh" - - name: generate_buildid_to_debug_symbols_mapping tags: ["symbolizer"] stepback: false @@ -8606,21 +8569,14 @@ tasks: args: - "./src/evergreen/generate_buildid_debug_symbols_mapping.sh" -- <<: *antithesis_config_build_and_push_template - name: antithesis_config_build_and_push_concurrency_sharded_with_stepdowns_and_balancer +- <<: *antithesis_task_template + name: antithesis_concurrency_sharded_with_stepdowns_and_balancer commands: - - func: "antithesis config image build and push" + - func: "do setup" + - func: "do setup for antithesis" + - func: "antithesis image build and push" vars: - suite: concurrency_sharded_with_stepdowns_and_balancer - -- <<: *antithesis_config_test_template - name: antithesis_config_test_concurrency_sharded_with_stepdowns_and_balancer - depends_on: - - name: antithesis_config_build_and_push_concurrency_sharded_with_stepdowns_and_balancer - commands: - - func: "antithesis config image test" - vars: - suite: concurrency_sharded_with_stepdowns_and_balancer + suite: antithesis_concurrency_sharded_with_stepdowns_and_balancer - <<: *task_template name: query_golden_classic @@ -9087,20 +9043,3 @@ task_groups: - "crypt_create_debug_lib" - "crypt_install_tests" - "crypt_run_tests" - -- name: antithesis_TG - max_hosts: 1 - share_processes: true - setup_group: - - func: "f_expansions_write" - - func: "git get project no modules" - - func: "f_expansions_write" - - func: "kill processes" - - func: "cleanup environment" - - func: "set up venv" - - func: "do setup" - setup_task: - - func: "set task expansion macros" - - func: "f_expansions_write" - tasks: - - .antithesis diff --git a/evergreen/antithesis_base_image_build_and_push.sh b/evergreen/antithesis_base_image_build_and_push.sh deleted file mode 100644 index 62c4f8b7e17..00000000000 --- a/evergreen/antithesis_base_image_build_and_push.sh +++ /dev/null @@ -1,58 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -set -euo pipefail - -# check that the binaries in dist-test are linked to libvoidstar -ldd src/dist-test/bin/mongod | grep libvoidstar -ldd src/dist-test/bin/mongos | grep libvoidstar -ldd src/dist-test/bin/mongo | grep libvoidstar - -# prepare the image building environment -cp -rf src/buildscripts/antithesis/ antithesis - -# copy ... to the build context -# resmoke -cp -rf src antithesis/base_images/workload/src -# mongo binary -cp src/dist-test/bin/mongo antithesis/base_images/workload -# libvoidstar -cp /usr/lib/libvoidstar.so antithesis/base_images/workload -# these aren't needed for the workload image, so get rid of them -rm -rf antithesis/base_images/workload/src/dist-test -# all mongodb binaries -cp -rf src/dist-test antithesis/base_images/mongo_binaries -cp /usr/lib/libvoidstar.so antithesis/base_images/mongo_binaries/ - -# push images as evergreen-latest-${branch_name}, unless it's a patch -tag="evergreen-latest-${branch_name}" -if [ "${is_patch}" = "true" ]; then - tag="evergreen-patch" -fi - -if [ -n "${antithesis_image_tag:-}" ]; then - echo "Using provided tag: '$antithesis_image_tag' for docker pushes" - tag=$antithesis_image_tag -fi - -# Build Image -cd antithesis/base_images/mongo_binaries -sudo docker build . -t mongo-binaries:$tag - -cd ../workload -sudo docker build . -t workload:$tag - -# Push Image -# login, push, and logout -echo "${antithesis_repo_key}" > mongodb.key.json -cat mongodb.key.json | sudo docker login -u _json_key https://us-central1-docker.pkg.dev --password-stdin -rm mongodb.key.json - -# tag and push to the registry -sudo docker tag "mongo-binaries:$tag" "us-central1-docker.pkg.dev/molten-verve-216720/mongodb-repository/mongo-binaries:$tag" -sudo docker push "us-central1-docker.pkg.dev/molten-verve-216720/mongodb-repository/mongo-binaries:$tag" - -sudo docker tag "workload:$tag" "us-central1-docker.pkg.dev/molten-verve-216720/mongodb-repository/workload:$tag" -sudo docker push "us-central1-docker.pkg.dev/molten-verve-216720/mongodb-repository/workload:$tag" - -sudo docker logout https://us-central1-docker.pkg.dev diff --git a/evergreen/antithesis_config_image_test.sh b/evergreen/antithesis_config_image_test.sh deleted file mode 100644 index 83ac6592e21..00000000000 --- a/evergreen/antithesis_config_image_test.sh +++ /dev/null @@ -1,9 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -set -o errexit -set -o verbose - -cd src/antithesis/antithesis_config/${suite} -sudo docker-compose up -d -sudo docker exec workload /bin/bash -c "cd resmoke && . python3-venv/bin/activate && python3 buildscripts/resmoke.py run --suite antithesis_${suite}" diff --git a/evergreen/antithesis_config_image_build_and_push.sh b/evergreen/antithesis_image_build_and_push.sh similarity index 51% rename from evergreen/antithesis_config_image_build_and_push.sh rename to evergreen/antithesis_image_build_and_push.sh index ad019207fad..aa70b4031fe 100644 --- a/evergreen/antithesis_config_image_build_and_push.sh +++ b/evergreen/antithesis_image_build_and_push.sh @@ -3,6 +3,9 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" set -o errexit +# The antithesis docker repository to push images to +antithesis_repo="us-central1-docker.pkg.dev/molten-verve-216720/mongodb-repository" + # push images as evergreen-latest-${branch_name}, unless it's a patch tag="evergreen-latest-${branch_name}" if [ "${is_patch}" = "true" ]; then @@ -17,12 +20,11 @@ fi # Build Image cd src activate_venv -$python buildscripts/resmoke.py generate-docker-compose --suite ${suite} -cp mongo-debugsymbols.tgz antithesis/antithesis_config/${suite}/debug +$python buildscripts/resmoke.py generate-docker-compose --in-evergreen --tag $tag ${suite} +# Test Image cd antithesis/antithesis_config/${suite} -sed -i s/evergreen-latest-master/$tag/ docker-compose.yml -sudo docker build . -t ${suite}-config:$tag +bash run_suite.sh # Push Image # login, push, and logout @@ -30,5 +32,13 @@ echo "${antithesis_repo_key}" > mongodb.key.json cat mongodb.key.json | sudo docker login -u _json_key https://us-central1-docker.pkg.dev --password-stdin rm mongodb.key.json -sudo docker tag "${suite}-config:$tag" "us-central1-docker.pkg.dev/molten-verve-216720/mongodb-repository/${suite}-config:$tag" -sudo docker push "us-central1-docker.pkg.dev/molten-verve-216720/mongodb-repository/${suite}-config:$tag" +sudo docker tag "${suite}:$tag" "$antithesis_repo/${suite}:$tag" +sudo docker push "$antithesis_repo/${suite}:$tag" + +sudo docker tag "mongo-binaries:$tag" "$antithesis_repo/mongo-binaries:$tag" +sudo docker push "$antithesis_repo/mongo-binaries:$tag" + +sudo docker tag "workload:$tag" "$antithesis_repo/workload:$tag" +sudo docker push "$antithesis_repo/workload:$tag" + +sudo docker logout https://us-central1-docker.pkg.dev diff --git a/evergreen/modify_debug_symbols.sh b/evergreen/modify_debug_symbols.sh index dcd1cf38863..134bb411c15 100644 --- a/evergreen/modify_debug_symbols.sh +++ b/evergreen/modify_debug_symbols.sh @@ -4,9 +4,13 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" set -o errexit cd src + # decompress debug symbols -tar -zxvf mongo-debugsymbols.tgz +mkdir debug_extract +tar -zxvf mongo-debugsymbols.tgz -C debug_extract + # renames dist-test to usr for Antithesis -mv dist-test usr +mv debug_extract/dist-test debug_extract/usr + # recompress debug symbols -tar -czvf mongo-debugsymbols.tgz usr +tar -czvf mongo-debugsymbols.tgz -C debug_extract usr