test_runner: add initial CLI runner

This commit introduces an initial version of a CLI-based test runner. PR-URL: #42658 Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>
nodejs · targos · May 3, 2022 · Apr 14, 2022 · Apr 14, 2022 · Apr 14, 2022
commit 78a860ae58bf332e9a8562233703a4eb4724f0db
diff --git a/doc/api/cli.md b/doc/api/cli.md
@@ -1052,6 +1052,16 @@ minimum allocation from the secure heap. The minimum value is `2`.
 The maximum value is the lesser of `--secure-heap` or `2147483647`.
 The value given must be a power of two.
 
+### `--test`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+Starts the Node.js command line test runner. This flag cannot be combined with
+`--check`, `--eval`, `--interactive`, or the inspector. See the documentation
+on [running tests from the command line][] for more details.
+
 ### `--test-only`
 
 <!-- YAML
@@ -2033,6 +2043,7 @@ $ node --max-old-space-size=1536 index.js
 [jitless]: https://v8.dev/blog/jitless
 [libuv threadpool documentation]: https://docs.libuv.org/en/latest/threadpool.html
 [remote code execution]: https://www.owasp.org/index.php/Code_Injection
+[running tests from the command line]: test.md#running-tests-from-the-command-line
 [security warning]: #warning-binding-inspector-to-a-public-ipport-combination-is-insecure
 [timezone IDs]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
 [ways that `TZ` is handled in other environments]: https://www.gnu.org/software/libc/manual/html_node/TZ-Variable.html
diff --git a/doc/api/test.md b/doc/api/test.md
@@ -219,6 +219,67 @@ test('a test that creates asynchronous activity', (t) => {
 });
 ```
 
+## Running tests from the command line
+
+The Node.js test runner can be invoked from the command line by passing the
+[`--test`][] flag:
+
+```bash
+node --test
+```
+
+By default, Node.js will recursively search the current directory for
+JavaScript source files matching a specific naming convention. Matching files
+are executed as test files. More information on the expected test file naming
+convention and behavior can be found in the [test runner execution model][]
+section.
+
+Alternatively, one or more paths can be provided as the final argument(s) to
+the Node.js command, as shown below.
+
+```bash
+node --test test1.js test2.mjs custom_test_dir/
+```
+
+In this example, the test runner will execute the files `test1.js` and
+`test2.mjs`. The test runner will also recursively search the
+`custom_test_dir/` directory for test files to execute.
+
+### Test runner execution model
+
+When searching for test files to execute, the test runner behaves as follows:
+
+* Any files explicitly provided by the user are executed.
+* If the user did not explicitly specify any paths, the current working
+  directory is recursively searched for files as specified in the following
+  steps.
+* `node_modules` directories are skipped unless explicitly provided by the
+  user.
+* If a directory named `test` is encountered, the test runner will search it
+  recursively for all all `.js`, `.cjs`, and `.mjs` files. All of these files
+  are treated as test files, and do not need to match the specific naming
+  convention detailed below. This is to accommodate projects that place all of
+  their tests in a single `test` directory.
+* In all other directories, `.js`, `.cjs`, and `.mjs` files matching the
+  following patterns are treated as test files:
+  * `^test$` - Files whose basename is the string `'test'`. Examples:
+    `test.js`, `test.cjs`, `test.mjs`.
+  * `^test-.+` - Files whose basename starts with the string `'test-'`
+    followed by one or more characters. Examples: `test-example.js`,
+    `test-another-example.mjs`.
+  * `.+[\.\-\_]test$` - Files whose basename ends with `.test`, `-test`, or
+    `_test`, preceded by one or more characters. Examples: `example.test.js`,
+    `example-test.cjs`, `example_test.mjs`.
+  * Other file types understood by Node.js such as `.node` and `.json` are not
+    automatically executed by the test runner, but are supported if explicitly
+    provided on the command line.
+
+Each matching test file is executed in a separate child process. If the child
+process finishes with an exit code of 0, the test is considered passing.
+Otherwise, the test is considered to be a failure. Test files must be
+executable by Node.js, but are not required to use the `node:test` module
+internally.
+
 ## `test([name][, options][, fn])`
 
 <!-- YAML
@@ -368,5 +429,7 @@ behaves in the same fashion as the top level [`test()`][] function.
 
 [TAP]: https://testanything.org/
 [`--test-only`]: cli.md#--test-only
+[`--test`]: cli.md#--test
 [`TestContext`]: #class-testcontext
 [`test()`]: #testname-options-fn
+[test runner execution model]: #test-runner-execution-model
diff --git a/doc/node.1 b/doc/node.1
@@ -381,6 +381,9 @@ the secure heap. The default is 0. The value must be a power of two.
 .It Fl -secure-heap-min Ns = Ns Ar n
 Specify the minimum allocation from the OpenSSL secure heap. The default is 2. The value must be a power of two.
 .
+.It Fl -test
+Starts the Node.js command line test runner.
+.
 .It Fl -test-only
 Configures the test runner to only execute top level tests that have the `only`
 option set.

diff --git a/lib/internal/errors.js b/lib/internal/errors.js
@@ -207,6 +207,16 @@ function isErrorStackTraceLimitWritable() {
     desc.set !== undefined;
 }
 
+function inspectWithNoCustomRetry(obj, options) {
+  const utilInspect = lazyInternalUtilInspect();
+
+  try {
+    return utilInspect.inspect(obj, options);
+  } catch {
+    return utilInspect.inspect(obj, { ...options, customInspect: false });
+  }
+}
+
 // A specialized Error that includes an additional info property with
 // additional information about the error condition.
 // It has the properties present in a UVException but with a custom error
@@ -862,6 +872,7 @@ module.exports = {
   getMessage,
   hideInternalStackFrames,
   hideStackFrames,
+  inspectWithNoCustomRetry,
   isErrorStackTraceLimitWritable,
   isStackOverflowError,
   kEnhanceStackBeforeInspector,
@@ -1549,11 +1560,14 @@ E('ERR_TEST_FAILURE', function(error, failureType) {
   assert(typeof failureType === 'string',
          "The 'failureType' argument must be of type string.");
 
-  const msg = error?.message ?? lazyInternalUtilInspect().inspect(error);
+  let msg = error?.message ?? error;
 
-  this.failureType = error?.failureType ?? failureType;
-  this.cause = error;
+  if (typeof msg !== 'string') {
+    msg = inspectWithNoCustomRetry(msg);
+  }
 
+  this.failureType = failureType;
+  this.cause = error;
   return msg;
 }, Error);
 E('ERR_TLS_CERT_ALTNAME_FORMAT', 'Invalid subject alternative name string',

diff --git a/lib/internal/main/test_runner.js b/lib/internal/main/test_runner.js
@@ -0,0 +1,157 @@
+'use strict';
+const {
+  ArrayFrom,
+  ArrayPrototypeFilter,
+  ArrayPrototypeIncludes,
+  ArrayPrototypePush,
+  ArrayPrototypeSlice,
+  ArrayPrototypeSort,
+  Promise,
+  SafeSet,
+} = primordials;
+const {
+  prepareMainThreadExecution,
+} = require('internal/bootstrap/pre_execution');
+const { spawn } = require('child_process');
+const { readdirSync, statSync } = require('fs');
+const console = require('internal/console/global');
+const {
+  codes: {
+    ERR_TEST_FAILURE,
+  },
+} = require('internal/errors');
+const test = require('internal/test_runner/harness');
+const { kSubtestsFailed } = require('internal/test_runner/test');
+const {
+  isSupportedFileType,
+  doesPathMatchFilter,
+} = require('internal/test_runner/utils');
+const { basename, join, resolve } = require('path');
+const kFilterArgs = ['--test'];
+
+prepareMainThreadExecution(false);
+markBootstrapComplete();
+
+// TODO(cjihrig): Replace this with recursive readdir once it lands.
+function processPath(path, testFiles, options) {
+  const stats = statSync(path);
+
+  if (stats.isFile()) {
+    if (options.userSupplied ||
+        (options.underTestDir && isSupportedFileType(path)) ||
+        doesPathMatchFilter(path)) {
+      testFiles.add(path);
+    }
+  } else if (stats.isDirectory()) {
+    const name = basename(path);
+
+    if (!options.userSupplied && name === 'node_modules') {
+      return;
+    }
+
+    // 'test' directories get special treatment. Recursively add all .js,
+    // .cjs, and .mjs files in the 'test' directory.
+    const isTestDir = name === 'test';
+    const { underTestDir } = options;
+    const entries = readdirSync(path);
+
+    if (isTestDir) {
+      options.underTestDir = true;
+    }
+
+    options.userSupplied = false;
+
+    for (let i = 0; i < entries.length; i++) {
+      processPath(join(path, entries[i]), testFiles, options);
+    }
+
+    options.underTestDir = underTestDir;
+  }
+}
+
+function createTestFileList() {
+  const cwd = process.cwd();
+  const hasUserSuppliedPaths = process.argv.length > 1;
+  const testPaths = hasUserSuppliedPaths ?
+    ArrayPrototypeSlice(process.argv, 1) : [cwd];
+  const testFiles = new SafeSet();
+
+  try {
+    for (let i = 0; i < testPaths.length; i++) {
+      const absolutePath = resolve(testPaths[i]);
+
+      processPath(absolutePath, testFiles, { userSupplied: true });
+    }
+  } catch (err) {
+    if (err?.code === 'ENOENT') {
+      console.error(`Could not find '${err.path}'`);
+      process.exit(1);
+    }
+
+    throw err;
+  }
+
+  return ArrayPrototypeSort(ArrayFrom(testFiles));
+}
+
+function filterExecArgv(arg) {
+  return !ArrayPrototypeIncludes(kFilterArgs, arg);
+}
+
+function runTestFile(path) {
+  return test(path, () => {
+    return new Promise((resolve, reject) => {
+      const args = ArrayPrototypeFilter(process.execArgv, filterExecArgv);
+      ArrayPrototypePush(args, path);
+
+      const child = spawn(process.execPath, args);
+      // TODO(cjihrig): Implement a TAP parser to read the child's stdout
+      // instead of just displaying it all if the child fails.
+      let stdout = '';
+      let stderr = '';
+      let err;
+
+      child.on('error', (error) => {
+        err = error;
+      });
+
+      child.stdout.setEncoding('utf8');
+      child.stderr.setEncoding('utf8');
+
+      child.stdout.on('data', (chunk) => {
+        stdout += chunk;
+      });
+
+      child.stderr.on('data', (chunk) => {
+        stderr += chunk;
+      });
+
+      child.once('exit', (code, signal) => {
+        if (code !== 0 || signal !== null) {
+          if (!err) {
+            err = new ERR_TEST_FAILURE('test failed', kSubtestsFailed);
+            err.exitCode = code;
+            err.signal = signal;
+            err.stdout = stdout;
+            err.stderr = stderr;
+            // The stack will not be useful since the failures came from tests
+            // in a child process.
+            err.stack = undefined;
+          }
+
+          return reject(err);
+        }
+
+        resolve();
+      });
+    });
+  });
+}
+
+(async function main() {
+  const testFiles = createTestFileList();
+
+  for (let i = 0; i < testFiles.length; i++) {
+    runTestFile(testFiles[i]);
+  }
+})();