stream: bump default highWaterMark

This should give a performance boost accross the board.

Given that the old limit is a decod old and memory capacity has
doubled many times since I think it is appropriate to slightly bump
the default limit.

PR-URL: https://github.com/nodejs/node/pull/52037
Refs: https://github.com/nodejs/node/pull/46608
Refs: https://github.com/nodejs/node/pull/50120
Reviewed-By: Rafael Gonzaga <rafael.nunu@hotmail.com>
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
Reviewed-By: Yagiz Nizipli <yagiz.nizipli@sentry.io>
Reviewed-By: Chengzhong Wu <legendecas@gmail.com>
Reviewed-By: Moshe Atlow <moshe@atlow.co.il>
Reviewed-By: Mohammed Keyvanzadeh <mohammadkeyvanzade94@gmail.com>
Reviewed-By: Trivikram Kamat <trivikr.dev@gmail.com>
Reviewed-By: Ruben Bridgewater <ruben@bridgewater.de>

benchmark/net/net-c2s.js

@@ -6,7 +6,7 @@ const net = require('net');
 const PORT = common.PORT;
 
 const bench = common.createBenchmark(main, {
-  len: [64, 102400, 1024 * 1024 * 16],
+  len: [64, 102400, 1024 * 64 * 16],
   type: ['utf', 'asc', 'buf'],
   dur: [5],
 }, {

benchmark/net/net-pipe.js

@@ -6,7 +6,7 @@ const net = require('net');
 const PORT = common.PORT;
 
 const bench = common.createBenchmark(main, {
-  len: [2, 64, 102400, 1024 * 1024 * 16],
+  len: [2, 64, 102400, 1024 * 64 * 16],
   type: ['utf', 'asc', 'buf'],
   dur: [5],
 }, {

benchmark/net/net-s2c.js

@@ -5,7 +5,7 @@ const common = require('../common.js');
 const PORT = common.PORT;
 
 const bench = common.createBenchmark(main, {
-  sendchunklen: [256, 32 * 1024, 128 * 1024, 16 * 1024 * 1024],
+  sendchunklen: [256, 32 * 1024, 128 * 1024, 16 * 64 * 1024],
   type: ['utf', 'asc', 'buf'],
   recvbuflen: [0, 64 * 1024, 1024 * 1024],
   recvbufgenfn: ['true', 'false'],

benchmark/net/net-wrap-js-stream-passthrough.js

@@ -5,7 +5,7 @@ const common = require('../common.js');
 const { PassThrough } = require('stream');
 
 const bench = common.createBenchmark(main, {
-  len: [64, 102400, 1024 * 1024 * 16],
+  len: [64, 102400, 1024 * 64 * 16],
   type: ['utf', 'asc', 'buf'],
   dur: [5],
 }, {

benchmark/net/tcp-raw-c2s.js

@@ -9,7 +9,7 @@ const util = require('util');
 // run the function with those settings.
 // if not, then queue up a bunch of child processes.
 const bench = common.createBenchmark(main, {
-  len: [102400, 1024 * 1024 * 16],
+  len: [102400, 1024 * 64 * 16],
   type: ['utf', 'asc', 'buf'],
   dur: [5],
 }, {

benchmark/net/tcp-raw-pipe.js

@@ -9,7 +9,7 @@ const util = require('util');
 // run the function with those settings.
 // if not, then queue up a bunch of child processes.
 const bench = common.createBenchmark(main, {
-  len: [102400, 1024 * 1024 * 16],
+  len: [102400, 1024 * 64 * 16],
   type: ['utf', 'asc', 'buf'],
   dur: [5],
 }, {

benchmark/net/tcp-raw-s2c.js

@@ -9,7 +9,7 @@ const util = require('util');
 // run the function with those settings.
 // If not, then queue up a bunch of child processes.
 const bench = common.createBenchmark(main, {
-  len: [102400, 1024 * 1024 * 16],
+  len: [102400, 1024 * 64 * 16],
   type: ['utf', 'asc', 'buf'],
   dur: [5],
 }, {

doc/api/stream.md

@@ -3479,6 +3479,9 @@ method.
 
 <!-- YAML
 changes:
+  - version: REPLACEME
+    pr-url: https://github.com/nodejs/node/pull/52037
+    description: bump default highWaterMark.
   - version: v15.5.0
     pr-url: https://github.com/nodejs/node/pull/36431
     description: support passing in an AbortSignal.
@@ -3500,7 +3503,7 @@ changes:
 * `options` {Object}
   * `highWaterMark` {number} Buffer level when
     [`stream.write()`][stream-write] starts returning `false`. **Default:**
-    `16384` (16 KiB), or `16` for `objectMode` streams.
+    `65536` (64 KiB), or `16` for `objectMode` streams.
   * `decodeStrings` {boolean} Whether to encode `string`s passed to
     [`stream.write()`][stream-write] to `Buffer`s (with the encoding
     specified in the [`stream.write()`][stream-write] call) before passing
@@ -3856,6 +3859,9 @@ constructor and implement the [`readable._read()`][] method.
 
 <!-- YAML
 changes:
+  - version: REPLACEME
+    pr-url: https://github.com/nodejs/node/pull/52037
+    description: bump default highWaterMark.
   - version: v15.5.0
     pr-url: https://github.com/nodejs/node/pull/36431
     description: support passing in an AbortSignal.
@@ -3873,7 +3879,7 @@ changes:
 * `options` {Object}
   * `highWaterMark` {number} The maximum [number of bytes][hwm-gotcha] to store
     in the internal buffer before ceasing to read from the underlying resource.
-    **Default:** `16384` (16 KiB), or `16` for `objectMode` streams.
+    **Default:** `65536` (64 KiB), or `16` for `objectMode` streams.
   * `encoding` {string} If specified, then buffers will be decoded to
     strings using the specified encoding. **Default:** `null`.
   * `objectMode` {boolean} Whether this stream should behave

lib/internal/streams/state.js

@@ -8,7 +8,8 @@ const { validateInteger } = require('internal/validators');
 
 const { ERR_INVALID_ARG_VALUE } = require('internal/errors').codes;
 
-let defaultHighWaterMarkBytes = 16 * 1024;
+// TODO (fix): For some reason Windows CI fails with bigger hwm.
+let defaultHighWaterMarkBytes = process.platform === 'win32' ? 16 * 1024 : 64 * 1024;
 let defaultHighWaterMarkObjectMode = 16;
 
 function highWaterMarkFrom(options, isDuplex, duplexKey) {

test/parallel/test-http-no-read-no-dump.js

@@ -34,7 +34,7 @@ const server = http.createServer((req, res) => {
   }, common.mustCall((res) => {
     res.resume();
 
-    post.write(Buffer.alloc(16 * 1024).fill('X'));
+    post.write(Buffer.alloc(64 * 1024).fill('X'));
     onPause = () => {
       post.end('something');
     };

test/parallel/test-https-hwm.js

@@ -62,5 +62,5 @@ const httpsServer = https.createServer({
     port: this.address().port,
     rejectUnauthorized: false,
     highWaterMark: undefined,
-  }, loadCallback(16 * 1024)).on('error', common.mustNotCall()).end();
+  }, loadCallback(process.platform === 'win32' ? 16 * 1024 : 64 * 1024)).on('error', common.mustNotCall()).end();
 }));

test/parallel/test-stream-duplex-readable-end.js

@@ -7,6 +7,7 @@ const stream = require('stream');
 let loops = 5;
 
 const src = new stream.Readable({
+  highWaterMark: 16 * 1024,
   read() {
     if (loops--)
       this.push(Buffer.alloc(20000));
@@ -14,6 +15,7 @@ const src = new stream.Readable({
 });
 
 const dst = new stream.Transform({
+  highWaterMark: 16 * 1024,
   transform(chunk, output, fn) {
     this.push(null);
     fn();

test/parallel/test-stream-pipe-await-drain-push-while-write.js

@@ -4,6 +4,7 @@ const stream = require('stream');
 const assert = require('assert');
 
 const writable = new stream.Writable({
+  highWaterMark: 16 * 1024,
   write: common.mustCall(function(chunk, encoding, cb) {
     assert.strictEqual(
       readable._readableState.awaitDrainWriters,
@@ -26,6 +27,7 @@ const writable = new stream.Writable({
 // A readable stream which produces two buffers.
 const bufs = [Buffer.alloc(32 * 1024), Buffer.alloc(33 * 1024)]; // above hwm
 const readable = new stream.Readable({
+  highWaterMark: 16 * 1024,
   read: function() {
     while (bufs.length > 0) {
       this.push(bufs.shift());

test/parallel/test-stream-readable-infinite-read.js

@@ -7,6 +7,7 @@ const { Readable } = require('stream');
 const buf = Buffer.alloc(8192);
 
 const readable = new Readable({
+  highWaterMark: 16 * 1024,
   read: common.mustCall(function() {
     this.push(buf);
   }, 31)

test/parallel/test-stream-transform-split-highwatermark.js

@@ -2,9 +2,9 @@
 require('../common');
 const assert = require('assert');
 
-const { Transform, Readable, Writable } = require('stream');
+const { Transform, Readable, Writable, getDefaultHighWaterMark } = require('stream');
 
-const DEFAULT = 16 * 1024;
+const DEFAULT = getDefaultHighWaterMark();
 
 function testTransform(expectedReadableHwm, expectedWritableHwm, options) {
   const t = new Transform(options);

test/parallel/test-stream-transform-split-objectmode.js

@@ -25,12 +25,14 @@ const assert = require('assert');
 
 const Transform = require('stream').Transform;
 
-const parser = new Transform({ readableObjectMode: true });
+const parser = new Transform({
+  readableObjectMode: true
+});
 
 assert(parser._readableState.objectMode);
 assert(!parser._writableState.objectMode);
 assert.strictEqual(parser.readableHighWaterMark, 16);
-assert.strictEqual(parser.writableHighWaterMark, 16 * 1024);
+assert.strictEqual(parser.writableHighWaterMark, process.platform === 'win32' ? 16 * 1024 : 64 * 1024);
 assert.strictEqual(parser.readableHighWaterMark,
                    parser._readableState.highWaterMark);
 assert.strictEqual(parser.writableHighWaterMark,
@@ -57,7 +59,7 @@ const serializer = new Transform({ writableObjectMode: true });
 
 assert(!serializer._readableState.objectMode);
 assert(serializer._writableState.objectMode);
-assert.strictEqual(serializer.readableHighWaterMark, 16 * 1024);
+assert.strictEqual(serializer.readableHighWaterMark, process.platform === 'win32' ? 16 * 1024 : 64 * 1024);
 assert.strictEqual(serializer.writableHighWaterMark, 16);
 assert.strictEqual(parser.readableHighWaterMark,
                    parser._readableState.highWaterMark);

test/parallel/test-tls-connect-hwm-option.js

@@ -37,7 +37,7 @@ server.listen(0, common.mustCall(() => {
     rejectUnauthorized: false,
     highWaterMark: undefined,
   }, common.mustCall(() => {
-    assert.strictEqual(defaultHighBob.readableHighWaterMark, 16 * 1024);
+    assert.strictEqual(defaultHighBob.readableHighWaterMark, process.platform === 'win32' ? 16 * 1024 : 64 * 1024);
     defaultHighBob.end();
   }));
 

test/parallel/test-zlib-brotli-16GB.js

@@ -3,6 +3,7 @@
 const common = require('../common');
 const { createBrotliDecompress } = require('node:zlib');
 const strictEqual = require('node:assert').strictEqual;
+const { getDefaultHighWaterMark } = require('stream');
 
 // This tiny HEX string is a 16GB file.
 // This test verifies that the stream actually stops.
@@ -18,5 +19,5 @@ decoder.end(buf);
 // to process the data and the buffer is not empty.
 setTimeout(common.mustCall(() => {
   // There is only one chunk in the buffer
-  strictEqual(decoder._readableState.buffer.length, 1);
+  strictEqual(decoder._readableState.buffer.length, getDefaultHighWaterMark() / (16 * 1024));
 }), common.platformTimeout(500));