SERVER-36352: Enable zstd compression on WiredTiger journal and collection files.

jstests/noPassthrough/block_compressor_options.js

@@ -0,0 +1,52 @@
+/**
+ * Tests using different combinations of --wiredTigerCollectionBlockCompressor and
+ * --wiredTigerJournalCompressor.
+ *
+ * Using the collection block compressor option will result in all new collections made during
+ * that process lifetime to use that compression setting. WiredTiger perfectly supports different
+ * tables using different block compressors. This test will start up MongoDB once for each block
+ * compressor setting and a create a new collection. Then after all collections are created, check
+ * creation string passed to WT via the collStats command.
+ *
+ * WiredTiger also supports changing the compression setting for writes to the journal. This tests
+ * that the setting can be changed between clean restarts, but otherwise does not verify the
+ * journal compression behavior.
+ *
+ * @tags: [requires_persistence,requires_wiredtiger]
+ */
+(function() {
+    'use strict';
+
+    // On the first iteration, start a mongod. Subsequent iterations will close and restart on the
+    // same dbpath.
+    let firstIteration = true;
+    let compressors = ['none', 'snappy', 'zlib', 'zstd'];
+    let mongo;
+    for (let compressor of compressors) {
+        jsTestLog({"Starting with compressor": compressor});
+        if (firstIteration) {
+            mongo = MongoRunner.runMongod({
+                wiredTigerCollectionBlockCompressor: compressor,
+                wiredTigerJournalCompressor: compressor
+            });
+            firstIteration = false;
+        } else {
+            MongoRunner.stopMongod(mongo);
+            mongo = MongoRunner.runMongod({
+                restart: true,
+                dbpath: mongo.dbpath,
+                cleanData: false,
+                wiredTigerCollectionBlockCompressor: compressor
+            });
+        }
+        mongo.getDB('db')[compressor].insert({});
+    }
+
+    for (let compressor of compressors) {
+        jsTestLog({"Asserting collection compressor": compressor});
+        let stats = mongo.getDB('db')[compressor].stats();
+        assert(stats['wiredTiger']['creationString'].search('block_compressor=' + compressor) > -1);
+    }
+
+    MongoRunner.stopMongod(mongo);
+}());

jstests/noPassthrough/compression_options.js

@@ -1,4 +1,4 @@
-// Checks storage-engine specific sections of db.severStatus() output.
+// Tests --networkMessageCompressors options.
 
 (function() {
     'use strict';

src/mongo/db/storage/wiredtiger/wiredtiger_global_options.cpp

@@ -65,8 +65,8 @@ Status WiredTigerGlobalOptions::add(moe::OptionSection* options) {
         .addOptionChaining("storage.wiredTiger.engineConfig.journalCompressor",
                            "wiredTigerJournalCompressor",
                            moe::String,
-                           "use a compressor for log records [none|snappy|zlib]")
-        .format("(:?none)|(:?snappy)|(:?zlib)", "(none/snappy/zlib)")
+                           "use a compressor for log records [none|snappy|zlib|zstd]")
+        .format("(:?none)|(:?snappy)|(:?zlib)|(:?zstd)", "(none/snappy/zlib/zstd)")
         .setDefault(moe::Value(std::string("snappy")));
     wiredTigerOptions.addOptionChaining("storage.wiredTiger.engineConfig.directoryForIndexes",
                                         "wiredTigerDirectoryForIndexes",
@@ -86,8 +86,8 @@ Status WiredTigerGlobalOptions::add(moe::OptionSection* options) {
                            "wiredTigerCollectionBlockCompressor",
                            moe::String,
                            "block compression algorithm for collection data "
-                           "[none|snappy|zlib]")
-        .format("(:?none)|(:?snappy)|(:?zlib)", "(none/snappy/zlib)")
+                           "[none|snappy|zlib|zstd]")
+        .format("(:?none)|(:?snappy)|(:?zlib)|(?:zstd)", "(none/snappy/zlib/zstd)")
         .setDefault(moe::Value(std::string("snappy")));
     wiredTigerOptions
         .addOptionChaining("storage.wiredTiger.collectionConfig.configString",

src/third_party/wiredtiger/SConscript

@@ -8,7 +8,7 @@ Import("endian")
 
 env = env.Clone()
 
-env.InjectThirdPartyIncludePaths(libraries=['snappy', 'zlib'])
+env.InjectThirdPartyIncludePaths(libraries=['snappy', 'zlib', 'zstd'])
 
 if endian == "big":
     env.Append(CPPDEFINES=[('WORDS_BIGENDIAN', 1)])
@@ -80,6 +80,7 @@ else:
 
 useZlib = True
 useSnappy = True
+useZstd = True
 
 version_file = 'build_posix/aclocal/version-set.m4'
 
@@ -181,6 +182,10 @@ if useSnappy:
     env.Append(CPPDEFINES=['HAVE_BUILTIN_EXTENSION_SNAPPY'])
     wtsources.append("ext/compressors/snappy/snappy_compress.c")
 
+if useZstd:
+    env.Append(CPPDEFINES=['HAVE_BUILTIN_EXTENSION_ZSTD'])
+    wtsources.append("ext/compressors/zstd/zstd_compress.c")
+
 # Use hardware by default on all platforms if available.
 # If not available at runtime, we fall back to software in some cases.
 if (get_option("use-hardware-crc32") == "off"):
@@ -199,6 +204,7 @@ wtlib = env.Library(
         '$BUILD_DIR/third_party/shim_allocator',
         '$BUILD_DIR/third_party/shim_snappy',
         '$BUILD_DIR/third_party/shim_zlib',
+        '$BUILD_DIR/third_party/shim_zstd',
     ],
     LIBDEPS_TAGS=[
         'init-no-global-side-effects',