SERVER-44364

SConstruct

@@ -1041,7 +1041,6 @@ envDict = dict(BUILD_ROOT=buildDir,
                CONFIGURELOG='$BUILD_ROOT/scons/config.log',
                CONFIG_HEADER_DEFINES={},
                LIBDEPS_TAG_EXPANSIONS=[],
-               AIB_PACKAGE_PREFIX='mongodb-',
                )
 
 env = Environment(variables=env_vars, **envDict)
@@ -3879,13 +3878,31 @@ if get_option('install-mode') == 'hygienic':
     env.AddPackageNameAlias(
         component="dist",
         role="runtime",
-        name="${{SERVER_DIST_BASENAME[{PREFIX_LEN}:]}}".format(PREFIX_LEN=len(env.get("AIB_PACKAGE_PREFIX")))
+        name="${SERVER_DIST_BASENAME}",
     )
 
     env.AddPackageNameAlias(
         component="dist",
         role="debug",
-        name="${{SERVER_DIST_BASENAME[{PREFIX_LEN}:]}}-debugsymbols".format(PREFIX_LEN=len(env.get("AIB_PACKAGE_PREFIX")))
+        name="${SERVER_DIST_BASENAME}-debugsymbols",
+    )
+
+    env.AddPackageNameAlias(
+        component="mh",
+        role="runtime",
+        # TODO: we should be able to move this to where the mqlrun binary is
+        # defined when AIB correctly uses environments instead of hooking into
+        # the first environment used.
+        name="${MH_DIST_BASENAME}",
+    )
+
+    env.AddPackageNameAlias(
+        component="mh",
+        role="debug",
+        # TODO: we should be able to move this to where the mqlrun binary is
+        # defined when AIB correctly uses environments instead of hooking into
+        # the first environment used.
+        name="${MH_DIST_BASENAME}-debugsymbols",
     )
 
     if env['PLATFORM'] == 'posix':
@@ -4084,6 +4101,7 @@ def add_version_to_distsrc(env, archive):
 env.AddDistSrcCallback(add_version_to_distsrc)
 
 env['SERVER_DIST_BASENAME'] = env.subst('mongodb-%s-$MONGO_DISTNAME' % (getSystemInstallName()))
+env['MH_DIST_BASENAME'] = env.subst('mh-%s-$MONGO_DISTNAME' % (getSystemInstallName()))
 if get_option('legacy-tarball') == 'true':
     if ('tar-dist' not in COMMAND_LINE_TARGETS and
         'zip-dist' not in COMMAND_LINE_TARGETS and
@@ -4314,9 +4332,13 @@ if get_option('install-mode') == 'hygienic':
     if env.TargetOSIs("windows"):
         env.Alias("archive-dist", "zip-dist")
         env.Alias("archive-dist-debug", "zip-dist-debug")
+        env.Alias("archive-mh", "zip-mh")
+        env.Alias("archive-mh-debug", "zip-mh-debug")
     else:
         env.Alias("archive-dist", "tar-dist")
         env.Alias("archive-dist-debug", "tar-dist-debug")
+        env.Alias("archive-mh", "tar-mh")
+        env.Alias("archive-mh-debug", "tar-mh-debug")
 
 # We don't want installing files to cause them to flow into the cache,
 # since presumably we can re-install them from the origin if needed.

etc/evergreen.yml

@@ -480,6 +480,15 @@ functions:
       bucket: mciuploads
       extract_to: src
 
+  "fetch packages": &fetch_packages
+    command: s3.get
+    params:
+      aws_key: ${aws_key}
+      aws_secret: ${aws_secret}
+      remote_file: ${project}/${build_variant}/${revision}/artifacts/${build_id}-packages.tgz
+      bucket: mciuploads
+      extract_to: src
+
   "fetch binaries": &fetch_binaries
     command: s3.get
     params:
@@ -1731,6 +1740,9 @@ functions:
           extra_args="$extra_args --cache-debug=scons_cache.log"
         fi
 
+        # Enable performance debugging
+        extra_args="$extra_args --debug=time"
+
         # If we are doing a patch build or we are building a non-push
         # build on the waterfall, then we don't need the --release
         # flag. Otherwise, this is potentially a build that "leaves
@@ -2407,7 +2419,7 @@ functions:
       script: |
         set -o errexit
 
-        export KITCHEN_ARTIFACTS_URL="https://s3.amazonaws.com/mciuploads/${project}/${build_variant}/${revision}/artifacts/${build_id}.tgz"
+        export KITCHEN_ARTIFACTS_URL="https://s3.amazonaws.com/mciuploads/${project}/${build_variant}/${revision}/artifacts/${build_id}-packages.tgz"
         export KITCHEN_SECURITY_GROUP="${kitchen_security_group}"
         export KITCHEN_SSH_KEY_ID="${kitchen_ssh_key_id}"
         export KITCHEN_SUBNET="${kitchen_subnet}"
@@ -3666,14 +3678,14 @@ tasks:
     - func: "build new tools"
     - func: "scons compile"
       vars:
-        targets: core tools ${mh_target|}
-        task_compile_flags: >-
-          --use-new-tools
-          --build-mongoreplay="${build_mongoreplay}"
-          --detect-odr-violations
-    - func: "scons compile"
-      vars:
-        targets: archive-dist archive-dist-debug distsrc-${ext|tgz} ${msi_target|}
+        targets: >-
+          archive-dist
+          archive-dist-debug
+          install-core
+          install-tools
+          distsrc-${ext|tgz}
+          ${msi_target|}
+          ${mh_target|}
         additional_targets: ""
         task_compile_flags: >-
           --use-new-tools
@@ -3697,7 +3709,8 @@ tasks:
           mv mongodb-src-*.${ext|tgz} distsrc.${ext|tgz}
           mv mongodb-*-debugsymbols.${ext|tgz} mongo-debugsymbols.tgz || true
           mv mongodb-*.${ext|tgz} mongodb-binaries.tgz
-          mv mh-*.${ext|tgz} mh-binaries.${ext|tgz} || true
+          mv mh-*-debugsymbols.${ext|tgz} mh-debugsymbols.${ext|tgz} || true
+          mv mh-mongodb*.${ext|tgz} mh-binaries.${ext|tgz} || true
 
     # Tar unstripped dbtest, to be archived in case of failure
     - command: archive.targz_pack
@@ -3707,26 +3720,6 @@ tasks:
         include:
           - "./dbtest*"
 
-    - command: shell.exec
-      type: test
-      params:
-        working_dir: src
-        script: |
-          set -o errexit
-          set -o verbose
-
-          if [ "${is_patch}" = "true" ] && [ "${bypass_compile|false}" = "true" ]; then
-            exit 0
-          fi
-
-          # If strip is on the path (everywhere except windows) then we should strip the test binaries
-          # before tarring them up
-          if [ -x ${strip_path|/usr/bin/strip} ]; then
-            ${strip_command|/usr/bin/strip} mongobridge
-            ${strip_command|/usr/bin/strip} wt
-          fi
-
-
     - command: shell.exec
       params:
         working_dir: src
@@ -3765,6 +3758,13 @@ tasks:
           fi
           cd ..
 
+    - command: archive.targz_pack
+      params:
+        target: "packages.tgz"
+        source_dir: "src"
+        include:
+          - "repo/**"
+
     - command: archive.targz_pack
       params:
         target: "artifacts.tgz"
@@ -3793,7 +3793,6 @@ tasks:
           - "./etc/*san.suppressions"
           - "./etc/repo_config.yaml"
           - "./build/**.gcno"
-          - "repo/**"
           - "src/mongo/util/options_parser/test_config_files/**"
           - "library_dependency_graph.json"
           - "src/third_party/JSON-Schema-Test-Suite/tests/draft4/**"
@@ -3802,6 +3801,7 @@ tasks:
           - "artifacts.json"
         exclude_files:
           - "*_test.pdb"
+
     - func: "upload debugsymbols"
     - command: s3.put
       params:
@@ -3835,7 +3835,18 @@ tasks:
         bucket: mciuploads
         permissions: public-read
         content_type: ${content_type|application/gzip}
-        display_name: MH
+        display_name: MH Binaries
+    - command: s3.put
+      params:
+        optional: true
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: src/mh-debugsymbols.${ext|tgz}
+        remote_file: ${mh_archive}
+        bucket: mciuploads
+        permissions: public-read
+        content_type: ${content_type|application/gzip}
+        display_name: MH Debuginfo
     - command: s3.put
       params:
         optional: true
@@ -3857,6 +3868,17 @@ tasks:
         permissions: public-read
         content_type: application/tar
         display_name: Artifacts
+    - command: s3.put
+      params:
+        optional: true
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: packages.tgz
+        remote_file: ${project}/${build_variant}/${revision}/artifacts/${build_id}-packages.tgz
+        bucket: mciuploads
+        permissions: public-read
+        content_type: application/tar
+        display_name: Packages
     - command: s3.put
       params:
         aws_key: ${aws_key}
@@ -3897,15 +3919,7 @@ tasks:
     - func: "build new tools"
     - func: "scons compile"
       vars:
-        targets: core tools ${mh_target|}
-        compiling_for_test: true
-        task_compile_flags: >-
-          --use-new-tools
-          --build-mongoreplay="${build_mongoreplay}"
-          --detect-odr-violations
-    - func: "scons compile"
-      vars:
-        targets: archive-dist
+        targets: install-core install-tools archive-dist ${mh_target|}
         compiling_for_test: true
         additional_targets: ""
         task_compile_flags: >-
@@ -8213,6 +8227,7 @@ tasks:
   - name: replica_sets_jscore_passthrough
   commands:
     - func: "fetch artifacts"
+    - func: "fetch packages"
     - func: "apply compile expansions"
     - func: "set up remote credentials"
       vars:
@@ -8243,6 +8258,7 @@ tasks:
   stepback: false
   commands:
     - func: "fetch artifacts"
+    - func: "fetch packages"
     - func: "fetch binaries"
     # Fetch the shell
     - command: s3.get
@@ -9160,7 +9176,7 @@ buildvariants:
     push_bucket: downloads.10gen.com
     push_name: linux
     push_arch: x86_64-enterprise-ubuntu1804
-    mh_target: dist-mh
+    mh_target: archive-mh archive-mh-debug
     compile_flags: --ssl MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
     multiversion_platform: ubuntu1804
     multiversion_edition: enterprise
@@ -9844,7 +9860,7 @@ buildvariants:
     platform_decompress: unzip
     exe: ".exe"
     msi_target: msi
-    mh_target: dist-mh
+    mh_target: archive-mh archive-mh-debug
     content_type: application/zip
     compile_flags: --ssl MONGO_DISTMOD=windows-64 CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --win-version-min=ws08r2
     # We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to
@@ -10291,7 +10307,7 @@ buildvariants:
     push_bucket: downloads.10gen.com
     push_name: macos
     push_arch: x86_64-enterprise
-    mh_target: dist-mh
+    mh_target: archive-mh archive-mh-debug
     compile_env: DEVELOPER_DIR=/Applications/Xcode10.2.app
     compile_flags: --ssl -j$(sysctl -n hw.logicalcpu) --libc++ --variables-files=etc/scons/xcode_macosx.vars
     resmoke_jobs_max: 6
@@ -10758,7 +10774,7 @@ buildvariants:
     push_bucket: downloads.10gen.com
     push_name: linux
     push_arch: x86_64-enterprise-rhel62
-    mh_target: dist-mh
+    mh_target: archive-mh archive-mh-debug
     compile_flags: --ssl MONGO_DISTMOD=rhel62 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
     multiversion_platform: rhel62
     multiversion_edition: enterprise
@@ -11198,7 +11214,7 @@ buildvariants:
     push_bucket: downloads.10gen.com
     push_name: linux
     push_arch: x86_64-enterprise-rhel70
-    mh_target: dist-mh
+    mh_target: archive-mh archive-mh-debug
     compile_flags: --ssl MONGO_DISTMOD=rhel70 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
     multiversion_platform: rhel70
     multiversion_edition: enterprise
@@ -11258,7 +11274,7 @@ buildvariants:
     push_bucket: downloads.10gen.com
     push_name: linux
     push_arch: x86_64-enterprise-rhel80
-    mh_target: dist-mh
+    mh_target: archive-mh archive-mh-debug
     compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
     multiversion_platform: rhel80
     multiversion_edition: enterprise
@@ -11820,7 +11836,7 @@ buildvariants:
     push_name: linux
     lang_environment: LANG=C
     push_arch: x86_64-enterprise-ubuntu1604
-    mh_target: dist-mh
+    mh_target: archive-mh archive-mh-debug
     compile_flags: --ssl MONGO_DISTMOD=ubuntu1604 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
     multiversion_platform: ubuntu1604
     multiversion_edition: enterprise
@@ -12203,7 +12219,7 @@ buildvariants:
     push_bucket: downloads.10gen.com
     push_name: linux
     push_arch: x86_64-enterprise-debian92
-    mh_target: dist-mh
+    mh_target: archive-mh archive-mh-debug
     compile_flags: --ssl MONGO_DISTMOD=debian92 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
     multiversion_platform: debian92
     multiversion_edition: enterprise
@@ -12307,7 +12323,7 @@ buildvariants:
     push_bucket: downloads.10gen.com
     push_name: linux
     push_arch: x86_64-enterprise-debian10
-    mh_target: dist-mh
+    mh_target: archive-mh archive-mh-debug
     compile_flags: --ssl MONGO_DISTMOD=debian10 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
     multiversion_platform: debian10
     multiversion_edition: enterprise
@@ -13081,7 +13097,7 @@ buildvariants:
   batchtime: 10080 # 7 days
   stepback: false
   expansions:
-    mh_target: dist-mh
+    mh_target: archive-mh archive-mh-debug
     compile_flags: --ssl MONGO_DISTMOD=rhel62 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
     multiversion_platform: rhel62
     multiversion_edition: enterprise

site_scons/site_tools/auto_install_binaries.py

@@ -36,7 +36,6 @@ COMPONENTS = "AIB_COMPONENTS_EXTRA"
 INSTALL_ACTIONS = "AIB_INSTALL_ACTIONS"
 META_ROLE = "AIB_META_ROLE"
 PACKAGE_ALIAS_MAP = "AIB_PACKAGE_ALIAS_MAP"
-PACKAGE_PREFIX = "AIB_PACKAGE_PREFIX"
 PRIMARY_COMPONENT = "AIB_COMPONENT"
 PRIMARY_ROLE = "AIB_ROLE"
 ROLES = "AIB_ROLES"
@@ -55,7 +54,7 @@ FILES should be absolute paths or relative to ROOT_DIRECTORY.
 
 ARCHIVE_TYPE is one of zip or tar.
 '''
-            
+
 if __name__ == "__main__":
     if len(sys.argv) < 4:
         print(sys.argv[0], "takes at minimum four arguments.")
@@ -252,7 +251,8 @@ def get_package_name(env, component, role):
         (component, role),
         "{component}-{role}".format(component=component, role=role)
     )
-    return "${{{prefix}}}{basename}".format(basename=basename, prefix=PACKAGE_PREFIX)
+
+    return basename
 
 
 def get_dependent_actions(
@@ -660,7 +660,7 @@ def dest_dir_generator(initial_value=None):
         if initial_value is None:
             dest_dir = env.Dir("#install")
         elif isinstance(initial_value, str):
-            dest_dir = env.Dir(initial_value) 
+            dest_dir = env.Dir(initial_value)
         elif isinstance(initial_value, SCons.Node.FS.Dir):
             dest_dir = initial_value
         else:
@@ -670,7 +670,7 @@ def dest_dir_generator(initial_value=None):
         return dd[1]
 
     return generator
-    
+
 
 def _aib_debugdir(source, target, env, for_signature):
     for s in source:
@@ -722,7 +722,6 @@ def generate(env):  # pylint: disable=too-many-statements
     env["PREFIX_DOCDIR"] = env.get("PREFIX_DOCDIR", "$PREFIX_SHAREDIR/doc")
     env["PREFIX_INCLUDEDIR"] = env.get("PREFIX_INCLUDEDIR", "$DESTDIR/include")
     env["PREFIX_DEBUGDIR"] = env.get("PREFIX_DEBUGDIR", _aib_debugdir)
-    env[PACKAGE_PREFIX] = env.get(PACKAGE_PREFIX, "")
     env[SUFFIX_MAP] = {}
     env[PACKAGE_ALIAS_MAP] = {}
     env[ALIAS_MAP] = defaultdict(dict)

src/mongo/embedded/SConscript

@@ -163,6 +163,7 @@ if get_option('link-model') != 'dynamic-sdk':
     env.Alias('all', mongoed) # This ensures it compiles and links, but doesn't copy it anywhere.
 
     hygienic = get_option('install-mode') == 'hygienic'
-
-    if not hygienic:
-        env.Install('#/', mongoed)
+    install_mongoed = env.Install('#/', mongoed)
+    if hygienic:
+        # TODO: remove when archive.tgz is built by hygienic
+        env.Alias("install-embedded-test", install_mongoed)

src/mongo/tools/SConscript

@@ -56,9 +56,9 @@ mongoebench = yamlEnv.Program(
     ],
 )
 
-hygienic = get_option('install-mode') == 'hygienic'
-if not hygienic:
-    env.Install("#/", mongobridge)
-    env.Install("#/", mongoebench)
+# TODO: remove this when hygienic is driving all tarball creation
+install_mongobridge = env.Install("#/", mongobridge)
+install_mongoebench = env.Install("#/", mongoebench)
+env.Alias("install-tools", [install_mongoebench, install_mongobridge])
 
 env.Alias('all', mongoebench)  # This ensures it compiles and links, but doesn't copy it anywhere.

src/third_party/wiredtiger/SConscript

@@ -232,6 +232,9 @@ wtbin = wtbinEnv.Program(
 )
 
 hygienic = get_option("install-mode") == "hygienic"
-
-if not hygienic:
-    wtbinEnv.Alias("tools", wtbinEnv.Install("#/", wtbin))
+wtbin_install = wtbinEnv.Install("#/", wtbin)
+if hygienic:
+    # TODO: remove this once hygienic drives all tarball creation
+    wtbinEnv.Alias("install-tools", wtbin_install)
+else:
+    wtbinEnv.Alias("tools", wtbin_install)