SERVER-63104 add resmoke argument for generating all_feature_flags.txt locally

buildscripts/idl/gen_all_feature_flag_list.py

@@ -30,7 +30,6 @@ Generate a file containing a list of disabled feature flags.
 Used by resmoke.py to run only feature flag tests.
 """
 
-import argparse
 import os
 import sys
 
@@ -43,6 +42,7 @@ sys.path.append(os.path.normpath(os.path.join(os.path.abspath(__file__), '../../
 
 # pylint: disable=wrong-import-position
 import buildscripts.idl.lib as lib
+from buildscripts.idl.idl import parser
 
 
 def is_third_party_idl(idl_path: str) -> bool:
@@ -56,13 +56,14 @@ def is_third_party_idl(idl_path: str) -> bool:
     return False
 
 
-def gen_all_feature_flags(idl_dir: str, import_dirs: List[str]):
+def gen_all_feature_flags(idl_dir: str = os.getcwd()):
     """Generate a list of all feature flags."""
     all_flags = []
     for idl_path in sorted(lib.list_idls(idl_dir)):
         if is_third_party_idl(idl_path):
             continue
-        for feature_flag in lib.parse_idl(idl_path, import_dirs).spec.feature_flags:
+        doc = parser.parse_file(open(idl_path), idl_path)
+        for feature_flag in doc.spec.feature_flags:
             if feature_flag.default.literal != "true":
                 all_flags.append(feature_flag.name)
 
@@ -72,18 +73,16 @@ def gen_all_feature_flags(idl_dir: str, import_dirs: List[str]):
     return list(set(all_flags) - set(force_disabled_flags))
 
 
+def gen_all_feature_flags_file(filename: str = lib.ALL_FEATURE_FLAG_FILE):
+    flags = gen_all_feature_flags()
+    with open(filename, "w") as output_file:
+        output_file.write("\n".join(flags))
+        print("Generated: ", os.path.realpath(output_file.name))
+
+
 def main():
     """Run the main function."""
-    arg_parser = argparse.ArgumentParser(description=__doc__)
-    arg_parser.add_argument("--import-dir", dest="import_dirs", type=str, action="append",
-                            help="Directory to search for IDL import files")
-
-    args = arg_parser.parse_args()
-
-    flags = gen_all_feature_flags(os.getcwd(), args.import_dirs)
-    with open(lib.ALL_FEATURE_FLAG_FILE, "w") as output_file:
-        for flag in flags:
-            output_file.write("%s\n" % flag)
+    gen_all_feature_flags_file()
 
 
 if __name__ == '__main__':

buildscripts/idl/idl/parser.py

@@ -1023,7 +1023,7 @@ def _propagate_globals(spec):
         idltype.cpp_type = _prefix_with_namespace(cpp_namespace, idltype.cpp_type)
 
 
-def _parse(stream, error_file_name):
+def parse_file(stream, error_file_name):
     # type: (Any, str) -> syntax.IDLParsedSpec
     """
     Parse a YAML document into an idl.syntax tree.
@@ -1125,7 +1125,7 @@ def parse(stream, input_file_name, resolver):
     input_file_name: a file name for error messages to use, and to help resolve imported files.
     """
 
-    root_doc = _parse(stream, input_file_name)
+    root_doc = parse_file(stream, input_file_name)
 
     if root_doc.errors:
         return root_doc
@@ -1162,7 +1162,7 @@ def parse(stream, input_file_name, resolver):
 
         # Parse imported file
         with resolver.open(resolved_file_name) as file_stream:
-            parsed_doc = _parse(file_stream, resolved_file_name)
+            parsed_doc = parse_file(file_stream, resolved_file_name)
 
         # Check for errors
         if parsed_doc.errors:

buildscripts/resmokelib/config.py

@@ -92,8 +92,9 @@ DEFAULTS = {
     "report_failure_status": "fail",
     "report_file": None,
     "run_all_feature_flag_tests": False,
-    "run_all_feature_flags_no_tests": False,
+    "run_no_feature_flag_tests": False,
     "additional_feature_flags": None,
+    "additional_feature_flags_file": None,
     "seed": int(time.time() * 256),  # Taken from random.py code in Python 2.7.
     "service_executor": None,
     "shell_conn_string": None,
@@ -379,8 +380,11 @@ INSTALL_DIR = None
 # Whether to run tests for feature flags.
 RUN_ALL_FEATURE_FLAG_TESTS = None
 
-# Whether to run the server with feature flags. Defaults to true if `RUN_ALL_FEATURE_FLAG_TESTS` is true.
-RUN_ALL_FEATURE_FLAGS = None
+# Whether to run the tests with enabled feature flags
+RUN_NO_FEATURE_FLAG_TESTS = None
+
+# the path to a file containing feature flags
+ADDITIONAL_FEATURE_FLAGS_FILE = None
 
 # List of enabled feature flags.
 ENABLED_FEATURE_FLAGS = []

buildscripts/resmokelib/configure_resmoke.py

@@ -15,6 +15,7 @@ import shlex
 
 import pymongo.uri_parser
 
+from buildscripts.idl import gen_all_feature_flag_list
 from buildscripts.idl.lib import ALL_FEATURE_FLAG_FILE
 
 from buildscripts.resmokelib import config as _config
@@ -52,14 +53,9 @@ def _validate_options(parser, args):
             "Cannot use --replayFile with additional test files listed on the command line invocation."
         )
 
-    if args.run_all_feature_flag_tests or args.run_all_feature_flags_no_tests:
-        if not os.path.isfile(ALL_FEATURE_FLAG_FILE):
-            parser.error(
-                "To run tests with all feature flags, the %s file must exist and be placed in"
-                " your working directory. The file can be downloaded from the artifacts tarball"
-                " in Evergreen. Alternatively, if you know which feature flags you want to enable,"
-                " you can use the --additionalFeatureFlags command line argument" %
-                ALL_FEATURE_FLAG_FILE)
+    if args.additional_feature_flags_file and not os.path.isfile(
+            args.additional_feature_flags_file):
+        parser.error("The specified additional feature flags file does not exist.")
 
     def get_set_param_errors(process_params):
         agg_set_params = collections.defaultdict(list)
@@ -185,28 +181,36 @@ be invoked as either:
 - buildscripts/resmoke.py --installDir {shlex.quote(user_config['install_dir'])}""")
         raise RuntimeError(err)
 
+    def process_feature_flag_file(path):
+        with open(path) as fd:
+            return fd.read().split()
+
     def setup_feature_flags():
         _config.RUN_ALL_FEATURE_FLAG_TESTS = config.pop("run_all_feature_flag_tests")
-        _config.RUN_ALL_FEATURE_FLAGS = config.pop("run_all_feature_flags_no_tests")
+        _config.RUN_NO_FEATURE_FLAG_TESTS = config.pop("run_no_feature_flag_tests")
+        _config.ADDITIONAL_FEATURE_FLAGS_FILE = config.pop("additional_feature_flags_file")
 
-        # Running all feature flag tests implies running the fixtures with feature flags.
         if _config.RUN_ALL_FEATURE_FLAG_TESTS:
-            _config.RUN_ALL_FEATURE_FLAGS = True
+            print("Generating: ", ALL_FEATURE_FLAG_FILE)
+            gen_all_feature_flag_list.gen_all_feature_flags_file()
 
         all_ff = []
         enabled_feature_flags = []
         try:
-            with open(ALL_FEATURE_FLAG_FILE) as fd:
-                all_ff = fd.read().split()
+            all_ff = process_feature_flag_file(ALL_FEATURE_FLAG_FILE)
         except FileNotFoundError:
             # If we ask resmoke to run with all feature flags, the feature flags file
             # needs to exist.
-            if _config.RUN_ALL_FEATURE_FLAGS:
+            if _config.RUN_ALL_FEATURE_FLAG_TESTS or _config.RUN_NO_FEATURE_FLAG_TESTS:
                 raise
 
-        if _config.RUN_ALL_FEATURE_FLAGS:
+        if _config.RUN_ALL_FEATURE_FLAG_TESTS:
             enabled_feature_flags = all_ff[:]
 
+        if _config.ADDITIONAL_FEATURE_FLAGS_FILE:
+            enabled_feature_flags.extend(
+                process_feature_flag_file(_config.ADDITIONAL_FEATURE_FLAGS_FILE))
+
         # Specify additional feature flags from the command line.
         # Set running all feature flag tests to True if this options is specified.
         additional_feature_flags = _tags_from_list(config.pop("additional_feature_flags"))
@@ -231,7 +235,7 @@ be invoked as either:
     _config.EXCLUDE_WITH_ANY_TAGS.extend(
         utils.default_if_none(_tags_from_list(config.pop("exclude_with_any_tags")), []))
 
-    if _config.RUN_ALL_FEATURE_FLAGS and not _config.RUN_ALL_FEATURE_FLAG_TESTS:
+    if _config.RUN_NO_FEATURE_FLAG_TESTS:
         # Don't run any feature flag tests.
         _config.EXCLUDE_WITH_ANY_TAGS.extend(all_feature_flags)
     else:

buildscripts/resmokelib/run/__init__.py

@@ -832,15 +832,19 @@ class RunPlugin(PluginInterface):
         )
 
         parser.add_argument(
-            "--runAllFeatureFlagsNoTests", dest="run_all_feature_flags_no_tests",
-            action="store_true", help=
-            "Run MongoDB servers with all feature flags enabled but don't run any tests tagged with these feature flags; used for multiversion suites"
-        )
+            "--runNoFeatureFlagTests", dest="run_no_feature_flag_tests", action="store_true",
+            help=("Do not run any tests tagged with enabled feature flags."
+                  " This argument has precedence over --runAllFeatureFlagTests"
+                  "; used for multiversion suites"))
 
         parser.add_argument("--additionalFeatureFlags", dest="additional_feature_flags",
                             action="append", metavar="featureFlag1, featureFlag2, ...",
                             help="Additional feature flags")
 
+        parser.add_argument("--additionalFeatureFlagsFile", dest="additional_feature_flags_file",
+                            action="store", metavar="FILE",
+                            help="The path to a file with feature flags, delimited by newlines.")
+
         parser.add_argument("--maxTestQueueSize", type=int, dest="max_test_queue_size",
                             help=argparse.SUPPRESS)
 
@@ -1188,15 +1192,6 @@ def to_local_args(input_args=None):
     if origin_suite is not None:
         setattr(parsed_args, "suite_files", origin_suite)
 
-    # Replace --runAllFeatureFlagTests with an explicit list of feature flags. The former relies on
-    # all_feature_flags.txt which may not exist in the local dev environment.
-    run_all_feature_flag_tests = getattr(parsed_args, "run_all_feature_flag_tests", None)
-    if run_all_feature_flag_tests is not None:
-        setattr(parsed_args, "additional_feature_flags", config.ENABLED_FEATURE_FLAGS)
-        del parsed_args.run_all_feature_flag_tests
-
-    del parsed_args.run_all_feature_flags_no_tests
-
     # The top-level parser has one subparser that contains all subcommand parsers.
     command_subparser = [
         action for action in parser._actions  # pylint: disable=protected-access

etc/evergreen.yml

@@ -1006,7 +1006,7 @@ buildvariants:
     # To force disable feature flags even on the all feature flags variant, please use this file:
     # buildscripts/resmokeconfig/fully_disabled_feature_flags.yml
     test_flags: >-
-      --runAllFeatureFlagTests
+      --additionalFeatureFlagsFile all_feature_flags.txt
       --excludeWithAnyTags=incompatible_with_windows_tls
       --excludeWithAnyTags=incompatible_with_shard_merge
     external_auth_jobs_max: 1
@@ -1596,7 +1596,7 @@ buildvariants:
     # To force disable feature flags even on the all feature flags variant, please use this file:
     # buildscripts/resmokeconfig/fully_disabled_feature_flags.yml
     test_flags: >-
-      --runAllFeatureFlagTests
+      --additionalFeatureFlagsFile all_feature_flags.txt
       --excludeWithAnyTags=incompatible_with_shard_merge
   tasks:
   - name: cqf
@@ -1892,7 +1892,7 @@ buildvariants:
     <<: *linux-x86-multiversion-expansions-template
     # No feature flag tests since they aren't compatible with the older binaries.
     test_flags: >-
-      --runAllFeatureFlagsNoTests
+      --runNoFeatureFlagTests
       --excludeWithAnyTags=incompatible_with_shard_merge
 
 - <<: *linux-x86-multiversion-template
@@ -2250,7 +2250,7 @@ buildvariants:
     idle_timeout_factor: 1.5
     exec_timeout_factor: 1.5
     test_flags: >-
-      --runAllFeatureFlagTests
+      --additionalFeatureFlagsFile all_feature_flags.txt
       --excludeWithAnyTags=incompatible_with_amazon_linux,incompatible_with_shard_merge
   tasks:
   - name: cqf
@@ -2384,7 +2384,7 @@ buildvariants:
     # buildscripts/resmokeconfig/fully_disabled_feature_flags.yml
     large_distro_name: rhel80-build
     test_flags: >-
-      --runAllFeatureFlagTests
+      --additionalFeatureFlagsFile all_feature_flags.txt
       --excludeWithAnyTags=incompatible_with_shard_merge
     separate_debug: off
   tasks:
@@ -2566,7 +2566,7 @@ buildvariants:
     test_flags: >-
       --excludeWithAnyTags=requires_ocsp_stapling
       --excludeWithAnyTags=incompatible_with_shard_merge
-      --runAllFeatureFlagTests
+      --additionalFeatureFlagsFile all_feature_flags.txt
     multiversion_platform: rhel80
     multiversion_edition: enterprise
     resmoke_jobs_factor: 0.3  # Avoid starting too many mongod's under UBSAN build.
@@ -2749,7 +2749,7 @@ buildvariants:
     test_flags: >-
       --excludeWithAnyTags=requires_ocsp_stapling
       --excludeWithAnyTags=incompatible_with_shard_merge
-      --runAllFeatureFlagTests
+      --additionalFeatureFlagsFile all_feature_flags.txt
   tasks:
   - name: jsCore
   - name: jsCore_txns
@@ -3461,7 +3461,7 @@ buildvariants:
     target_resmoke_time: 30
     max_sub_suites: 3
     test_flags: >-
-      --runAllFeatureFlagTests
+      --additionalFeatureFlagsFile all_feature_flags.txt
       --excludeWithAnyTags=resource_intensive
       --excludeWithAnyTags=incompatible_with_shard_merge
 

etc/perf.yml

@@ -320,7 +320,7 @@ functions:
           set -o errexit
           set -o verbose
           source "${workdir}/compile_venv/bin/activate"
-          python ./buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir src/mongo/db/modules/enterprise/src
+          python ./buildscripts/idl/gen_all_feature_flag_list.py
           mkdir -p mongodb/feature_flags
           cp ./all_feature_flags.txt mongodb/feature_flags
     - command: shell.exec

etc/system_perf.yml

@@ -349,7 +349,7 @@ functions:
           set -o errexit
           set -o verbose
           source "${workdir}/compile_venv/bin/activate"
-          python ./buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir src/mongo/db/modules/enterprise/src
+          python ./buildscripts/idl/gen_all_feature_flag_list.py
           mkdir -p mongodb/feature_flags
           cp ./all_feature_flags.txt mongodb/feature_flags
     - command: shell.exec

evergreen/feature_flag_tags_check.sh

@@ -12,7 +12,7 @@ enterprise_path="src/mongo/db/modules/enterprise"
 diff_file_name="with_base_upstream.diff"
 
 # get the list of feature flags from the patched version
-$python buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir "$enterprise_path"/src
+$python buildscripts/idl/gen_all_feature_flag_list.py
 mv all_feature_flags.txt patch_all_feature_flags.txt
 
 # get the list of feature flags from the base commit
@@ -29,7 +29,7 @@ if [ -s "$diff_file_name" ]; then
 fi
 popd
 
-$python buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir "$enterprise_path"/src
+$python buildscripts/idl/gen_all_feature_flag_list.py
 mv all_feature_flags.txt base_all_feature_flags.txt
 
 # print out the list of tests that previously had feature flag tag, that was

evergreen/gen_feature_flags.sh

@@ -6,4 +6,4 @@ cd src
 set -o errexit
 set -o verbose
 activate_venv
-$python buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir src/mongo/db/modules/enterprise/src
+$python buildscripts/idl/gen_all_feature_flag_list.py