SERVER-41672 Use hygienic to produce dist and distsrc archives in evergreen

SConstruct

@@ -3795,7 +3795,14 @@ if get_option('install-mode') == 'hygienic':
                 "debug"
             ]
         ),
-        
+
+        ".pdb": env.SuffixMap(
+            directory="$PREFIX_DEBUGDIR",
+            default_roles=[
+                "debug"
+            ]
+        ),
+
         ".lib": env.SuffixMap(
             directory="$PREFIX_LIBDIR",
             default_roles=[
@@ -3817,6 +3824,12 @@ if get_option('install-mode') == 'hygienic':
         name="${{SERVER_DIST_BASENAME[{PREFIX_LEN}:]}}".format(PREFIX_LEN=len(env.get("AIB_PACKAGE_PREFIX")))
     )
 
+    env.AddPackageNameAlias(
+        component="dist",
+        role="debug",
+        name="${{SERVER_DIST_BASENAME[{PREFIX_LEN}:]}}-debugsymbols".format(PREFIX_LEN=len(env.get("AIB_PACKAGE_PREFIX")))
+    )
+
     if env['PLATFORM'] == 'posix':
         env.AppendUnique(
             RPATH=[
@@ -4185,8 +4198,10 @@ if get_option('install-mode') == 'hygienic':
     # the evergreen.yml make this decision
     if env.TargetOSIs("windows"):
         env.Alias("archive-dist", "zip-dist")
+        env.Alias("archive-dist-debug", "zip-dist-debug")
     else:
         env.Alias("archive-dist", "tar-dist")
+        env.Alias("archive-dist-debug", "tar-dist-debug")
 
 # We don't want installing files to cause them to flow into the cache,	
 # since presumably we can re-install them from the origin if needed.	

etc/evergreen.yml

@@ -3574,11 +3574,22 @@ tasks:
     - func: "build new tools"
     - func: "scons compile"
       vars:
-        targets: core tools dist dist-debugsymbols distsrc-${ext|tgz} ${msi_target|} ${mh_target|}
+        targets: core tools ${mh_target|}
         task_compile_flags: >-
           --use-new-tools
           --build-mongoreplay="${build_mongoreplay}"
           --detect-odr-violations
+    - func: "scons compile"
+      vars:
+        targets: archive-dist archive-dist-debug distsrc-${ext|tgz} ${msi_target|}
+        additional_targets: ""
+        task_compile_flags: >-
+          --use-new-tools
+          --build-mongoreplay="${build_mongoreplay}"
+          --detect-odr-violations
+          --install-mode=hygienic
+          --separate-debug
+          --legacy-tarball
     - command: shell.exec
       type: test
       params:

site_scons/site_tools/auto_install_binaries.py

@@ -74,7 +74,7 @@ if __name__ == "__main__":
         import subprocess
         import shlex
         tar = which("tar")
-        tar_cmd = "{tar} -C {root_dir} -cf {archive_name} {files}".format(
+        tar_cmd = "{tar} -C {root_dir} -czf {archive_name} {files}".format(
             tar=tar,
             root_dir=root_dir,
             archive_name=archive_name,
@@ -84,8 +84,8 @@ if __name__ == "__main__":
         sys.exit(0)
 
     if archive_type == "zip":
-        from zipfile import ZipFile
-        archive = ZipFile(archive_name, mode='w')
+        import zipfile
+        archive = zipfile.ZipFile(archive_name, mode='w', compression=zipfile.ZIP_DEFLATED)
         add_file = archive.write
     else:
         import tarfile
@@ -610,8 +610,12 @@ def generate(env):  # pylint: disable=too-many-statements
     # TODO: make this configurable?
     env[ROLE_DEPENDENCIES] = {
         "debug": [
-            "runtime",
-            "base",
+            # TODO: Debug should depend on these when making packages, but shouldn't when building
+            # the legacy tarball. Probably fuel for the above configurability fire. For now, make it not
+            # depend so that we can get AIB in place for the dist builders.
+
+            # "runtime",
+            # "base",
         ],
         "dev": [
             "runtime",

site_scons/site_tools/separate_debug.py

@@ -85,6 +85,8 @@ def _update_builder(env, builder, bitcode):
                 bitcode_file = env.File(str(target[0]) + ".bcsymbolmap")
         elif env.TargetOSIs('posix'):
             debug_file = env.File(str(target[0]) + ".debug")
+        elif env.TargetOSIs('windows'):
+            debug_file = env.File(env.subst('${PDB}', target=target))
         else:
             pass
 

src/mongo/SConscript

@@ -809,23 +809,12 @@ else:
                                  distsrc.File('THIRD-PARTY-NOTICES'),
                                  distsrc.File('MPL-2')])
 
-enterprise_license = [banner for banner in env["MODULE_BANNERS"] if banner.name ==  "LICENSE-Enterprise.txt"]
-if hygienic:
-    env.AutoInstall(
-        target='$DESTDIR',
-        source=[
-            enterprise_license[0]
-            if enterprise_license else distsrc.File('LICENSE-Community.txt'),
-        ],
-        AIB_COMPONENT='common',
-        AIB_COMPONENTS_EXTRA=['dist'],
-        AIB_ROLE='base',
-    )
 # If no module has introduced a file named LICENSE-Enterprise.txt then this
 # is a Community build, so inject the AGPL and the Community license
-elif not enterprise_license:
+
+enterprise_license = [banner for banner in env["MODULE_BANNERS"] if banner.name ==  "LICENSE-Enterprise.txt"]
+if not enterprise_license:
     env.Append(MODULE_BANNERS = [distsrc.File('LICENSE-Community.txt')])
-    
 
 # All module banners get staged to the top level of the tarfile, so we
 # need to fail if we are going to have a name collision.
@@ -834,6 +823,15 @@ if not len(module_banner_filenames) == len(env['MODULE_BANNERS']):
     # TODO: Be nice and identify conflicts in error.
     env.FatalError("ERROR: Filename conflicts exist in module banners.")
 
+if hygienic:
+    env.AutoInstall(
+        target='$DESTDIR',
+        source=env.get('MODULE_BANNERS', []),
+        AIB_COMPONENT='common',
+        AIB_COMPONENTS_EXTRA=['dist'],
+        AIB_ROLE='base',
+    )
+
 # Build a set of directories containing module banners, and use that
 # to build a --transform option for each directory so that the files
 # are tar'ed up to the proper location.
@@ -914,29 +912,30 @@ if has_option("use-new-tools") and not hygienic:
 
     env.Alias("dist", server_archive)
     env.NoCache(server_archive)
+
+    debug_symbols_dist = env.Command(
+        target='#/${SERVER_DIST_BASENAME}-debugsymbols${DIST_ARCHIVE_SUFFIX}',
+        source=['#buildscripts/make_archive.py'] + distDebugSymbols,
+        action=' '.join(
+            [
+                '$PYTHON ${SOURCES[0]} -o $TARGET',
+                '--transform $BUILD_DIR/mongo/db/modules/enterprise=$SERVER_DIST_BASENAME',
+                '--transform $BUILD_DIR/mongo=$SERVER_DIST_BASENAME',
+                '${TEMPFILE(SOURCES[1:])}',
+            ]
+        ),
+        BUILD_DIR=env.Dir('$BUILD_DIR').path
+    )
+
+    env.Alias('dist-debugsymbols', debug_symbols_dist)
+    env.NoCache(debug_symbols_dist)
+
 elif not hygienic:
     def failDist(env, target, source):
         env.FatalError("ERROR: 'dist' target only valid with --use-new-tools.")
     env.Alias("dist", [], [ failDist ] )
     env.AlwaysBuild("dist")
 
-debug_symbols_dist = env.Command(
-    target='#/${SERVER_DIST_BASENAME}-debugsymbols${DIST_ARCHIVE_SUFFIX}',
-    source=['#buildscripts/make_archive.py'] + distDebugSymbols,
-    action=' '.join(
-        [
-            '$PYTHON ${SOURCES[0]} -o $TARGET',
-            '--transform $BUILD_DIR/mongo/db/modules/enterprise=$SERVER_DIST_BASENAME',
-            '--transform $BUILD_DIR/mongo=$SERVER_DIST_BASENAME',
-            '${TEMPFILE(SOURCES[1:])}',
-        ]
-    ),
-    BUILD_DIR=env.Dir('$BUILD_DIR').path
-)
-
-env.Alias('dist-debugsymbols', debug_symbols_dist)
-env.NoCache(debug_symbols_dist)
-
 #final alias
 if not hygienic:
     env.Alias( "install", "$DESTDIR" )