GeezerVison is showing github.com — test_runner: support expecting a test-case to fail · nodejs/node@ff1fcab
File treeExpand file treeCollapse file tree Open diff view settings test/fixtures/test-runner/output
Expand file treeCollapse file tree Open diff view settings Collapse file Copy file name to clipboardExpand all lines: doc/api/test.md +51Lines changed: 51 additions & 0 deletions - Display the source diff
- Display the rich diff
| Original file line number | Diff line number | Diff line change |
|---|
@@ -224,6 +224,57 @@ test('todo() method with message', (t) => { | 224 | 224 | }); | 225 | 225 | ``` | 226 | 226 |
| | 227 | +## Expecting tests to fail | | 228 | + | | 229 | +<!-- YAML | | 230 | +added: | | 231 | + - REPLACEME | | 232 | +--> | | 233 | + | | 234 | +This flips the pass/fail reporting for a specific test or suite: A flagged test/test-case must throw | | 235 | +in order to "pass"; a test/test-case that does not throw, fails. | | 236 | + | | 237 | +In the following, `doTheThing()` returns _currently_ `false` (`false` does not equal `true`, causing | | 238 | +`strictEqual` to throw, so the test-case passes). | | 239 | + | | 240 | +```js | | 241 | +it.expectFailure('should do the thing', () => { | | 242 | + assert.strictEqual(doTheThing(), true); | | 243 | +}); | | 244 | + | | 245 | +it('should do the thing', { expectFailure: true }, () => { | | 246 | + assert.strictEqual(doTheThing(), true); | | 247 | +}); | | 248 | +``` | | 249 | + | | 250 | +`skip` and/or `todo` are mutually exclusive to `expectFailure`, and `skip` or `todo` | | 251 | +will "win" when both are applied (`skip` wins against both, and `todo` wins | | 252 | +against `expectFailure`). | | 253 | + | | 254 | +These tests will be skipped (and not run): | | 255 | + | | 256 | +```js | | 257 | +it.expectFailure('should do the thing', { skip: true }, () => { | | 258 | + assert.strictEqual(doTheThing(), true); | | 259 | +}); | | 260 | + | | 261 | +it.skip('should do the thing', { expectFailure: true }, () => { | | 262 | + assert.strictEqual(doTheThing(), true); | | 263 | +}); | | 264 | +``` | | 265 | + | | 266 | +These tests will be marked "todo" (silencing errors): | | 267 | + | | 268 | +```js | | 269 | +it.expectFailure('should do the thing', { todo: true }, () => { | | 270 | + assert.strictEqual(doTheThing(), true); | | 271 | +}); | | 272 | + | | 273 | +it.todo('should do the thing', { expectFailure: true }, () => { | | 274 | + assert.strictEqual(doTheThing(), true); | | 275 | +}); | | 276 | +``` | | 277 | + | 227 | 278 | ## `describe()` and `it()` aliases | 228 | 279 |
| 229 | 280 | Suites and tests can also be written using the `describe()` and `it()` | |
Collapse file Copy file name to clipboardExpand all lines: lib/internal/test_runner/harness.js +1-1Lines changed: 1 addition & 1 deletion | Original file line number | Diff line number | Diff line change |
|---|
@@ -377,7 +377,7 @@ function runInParentContext(Factory) { | 377 | 377 |
| 378 | 378 | return run(name, options, fn, overrides); | 379 | 379 | }; | 380 | | - ArrayPrototypeForEach(['skip', 'todo', 'only'], (keyword) => { | | 380 | + ArrayPrototypeForEach(['expectFailure', 'skip', 'todo', 'only'], (keyword) => { | 381 | 381 | test[keyword] = (name, options, fn) => { | 382 | 382 | const overrides = { | 383 | 383 | __proto__: null, | |
Collapse file Copy file name to clipboardExpand all lines: lib/internal/test_runner/reporter/tap.js +5-3Lines changed: 5 additions & 3 deletions | Original file line number | Diff line number | Diff line change |
|---|
@@ -33,12 +33,12 @@ async function * tapReporter(source) { | 33 | 33 | for await (const { type, data } of source) { | 34 | 34 | switch (type) { | 35 | 35 | case 'test:fail': { | 36 | | - yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo); | | 36 | + yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure); | 37 | 37 | const location = data.file ? `${data.file}:${data.line}:${data.column}` : null; | 38 | 38 | yield reportDetails(data.nesting, data.details, location); | 39 | 39 | break; | 40 | 40 | } case 'test:pass': | 41 | | - yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo); | | 41 | + yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure); | 42 | 42 | yield reportDetails(data.nesting, data.details, null); | 43 | 43 | break; | 44 | 44 | case 'test:plan': | @@ -65,7 +65,7 @@ async function * tapReporter(source) { | 65 | 65 | } | 66 | 66 | } | 67 | 67 |
| 68 | | -function reportTest(nesting, testNumber, status, name, skip, todo) { | | 68 | +function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure) { | 69 | 69 | let line = `${indent(nesting)}${status} ${testNumber}`; | 70 | 70 |
| 71 | 71 | if (name) { | @@ -76,6 +76,8 @@ function reportTest(nesting, testNumber, status, name, skip, todo) { | 76 | 76 | line += ` # SKIP${typeof skip === 'string' && skip.length ? ` ${tapEscape(skip)}` : ''}`; | 77 | 77 | } else if (todo !== undefined) { | 78 | 78 | line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`; | | 79 | + } else if (expectFailure !== undefined) { | | 80 | + line += ' # EXPECTED FAILURE'; | 79 | 81 | } | 80 | 82 |
| 81 | 83 | line += '\n'; | |
Collapse file Copy file name to clipboardExpand all lines: lib/internal/test_runner/reporter/utils.js +3-1Lines changed: 3 additions & 1 deletion | Original file line number | Diff line number | Diff line change |
|---|
@@ -71,14 +71,16 @@ function formatError(error, indent) { | 71 | 71 | function formatTestReport(type, data, prefix = '', indent = '', hasChildren = false, showErrorDetails = true) { | 72 | 72 | let color = reporterColorMap[type] ?? colors.white; | 73 | 73 | let symbol = reporterUnicodeSymbolMap[type] ?? ' '; | 74 | | - const { skip, todo } = data; | | 74 | + const { skip, todo, expectFailure } = data; | 75 | 75 | const duration_ms = data.details?.duration_ms ? ` ${colors.gray}(${data.details.duration_ms}ms)${colors.white}` : ''; | 76 | 76 | let title = `${data.name}${duration_ms}`; | 77 | 77 |
| 78 | 78 | if (skip !== undefined) { | 79 | 79 | title += ` # ${typeof skip === 'string' && skip.length ? skip : 'SKIP'}`; | 80 | 80 | } else if (todo !== undefined) { | 81 | 81 | title += ` # ${typeof todo === 'string' && todo.length ? todo : 'TODO'}`; | | 82 | + } else if (expectFailure !== undefined) { | | 83 | + title += ` # EXPECTED FAILURE`; | 82 | 84 | } | 83 | 85 |
| 84 | 86 | const error = showErrorDetails ? formatError(data.details?.error, indent) : ''; | |
Collapse file Copy file name to clipboardExpand all lines: lib/internal/test_runner/test.js +11-2Lines changed: 11 additions & 2 deletions | Original file line number | Diff line number | Diff line change |
|---|
@@ -496,7 +496,7 @@ class Test extends AsyncResource { | 496 | 496 | super('Test'); | 497 | 497 |
| 498 | 498 | let { fn, name, parent } = options; | 499 | | - const { concurrency, entryFile, loc, only, timeout, todo, skip, signal, plan } = options; | | 499 | + const { concurrency, entryFile, expectFailure, loc, only, timeout, todo, skip, signal, plan } = options; | 500 | 500 |
| 501 | 501 | if (typeof fn !== 'function') { | 502 | 502 | fn = noop; | @@ -635,6 +635,7 @@ class Test extends AsyncResource { | 635 | 635 | this.plan = null; | 636 | 636 | this.expectedAssertions = plan; | 637 | 637 | this.cancelled = false; | | 638 | + this.expectFailure = expectFailure !== undefined && expectFailure !== false; | 638 | 639 | this.skipped = skip !== undefined && skip !== false; | 639 | 640 | this.isTodo = (todo !== undefined && todo !== false) || this.parent?.isTodo; | 640 | 641 | this.startTime = null; | @@ -938,7 +939,12 @@ class Test extends AsyncResource { | 938 | 939 | return; | 939 | 940 | } | 940 | 941 |
| 941 | | - this.passed = false; | | 942 | + if (this.expectFailure === true) { | | 943 | + this.passed = true; | | 944 | + } else { | | 945 | + this.passed = false; | | 946 | + } | | 947 | + | 942 | 948 | this.error = err; | 943 | 949 | } | 944 | 950 |
| @@ -1335,6 +1341,8 @@ class Test extends AsyncResource { | 1335 | 1341 | directive = this.reporter.getSkip(this.message); | 1336 | 1342 | } else if (this.isTodo) { | 1337 | 1343 | directive = this.reporter.getTodo(this.message); | | 1344 | + } else if (this.expectFailure) { | | 1345 | + directive = this.reporter.getXFail(this.expectFailure); // TODO(@JakobJingleheimer): support specifying failure | 1338 | 1346 | } | 1339 | 1347 |
| 1340 | 1348 | if (this.reportedType) { | @@ -1349,6 +1357,7 @@ class Test extends AsyncResource { | 1349 | 1357 | if (this.passedAttempt !== undefined) { | 1350 | 1358 | details.passed_on_attempt = this.passedAttempt; | 1351 | 1359 | } | | 1360 | + | 1352 | 1361 | return { __proto__: null, details, directive }; | 1353 | 1362 | } | 1354 | 1363 |
| |
Collapse file Copy file name to clipboardExpand all lines: lib/internal/test_runner/tests_stream.js +4Lines changed: 4 additions & 0 deletions | Original file line number | Diff line number | Diff line change |
|---|
@@ -87,6 +87,10 @@ class TestsStream extends Readable { | 87 | 87 | return { __proto__: null, todo: reason ?? true }; | 88 | 88 | } | 89 | 89 |
| | 90 | + getXFail(expectation = undefined) { | | 91 | + return { __proto__: null, expectFailure: expectation ?? true }; | | 92 | + } | | 93 | + | 90 | 94 | enqueue(nesting, loc, name, type) { | 91 | 95 | this[kEmitMessage]('test:enqueue', { | 92 | 96 | __proto__: null, | |
Collapse file Copy file name to clipboardExpand all lines: test/fixtures/test-runner/output/describe_it.js +24-1Lines changed: 24 additions & 1 deletion | Original file line number | Diff line number | Diff line change |
|---|
@@ -5,8 +5,23 @@ const { describe, it, test } = require('node:test'); | 5 | 5 | const util = require('util'); | 6 | 6 |
| 7 | 7 |
| 8 | | -it.todo('sync pass todo', () => { | | 8 | +it.expectFailure('sync expect fail (method)', () => { | | 9 | + throw new Error('should pass'); | | 10 | +}); | | 11 | + | | 12 | +it('sync expect fail (options)', { expectFailure: true }, () => { | | 13 | + throw new Error('should pass'); | | 14 | +}); | 9 | 15 |
| | 16 | +it.expectFailure('async expect fail (method)', async () => { | | 17 | + throw new Error('should pass'); | | 18 | +}); | | 19 | + | | 20 | +it('async expect fail (options)', { expectFailure: true }, async () => { | | 21 | + throw new Error('should pass'); | | 22 | +}); | | 23 | + | | 24 | +it.todo('sync pass todo', () => { | 10 | 25 | }); | 11 | 26 |
| 12 | 27 | it('sync pass todo with message', { todo: 'this is a passing todo' }, () => { | @@ -16,13 +31,21 @@ it.todo('sync todo', () => { | 16 | 31 | throw new Error('should not count as a failure'); | 17 | 32 | }); | 18 | 33 |
| | 34 | +it.todo('sync todo with expect fail', { expectFailure: true }, () => { | | 35 | + throw new Error('should not count as an expected failure'); | | 36 | +}); | | 37 | + | 19 | 38 | it('sync todo with message', { todo: 'this is a failing todo' }, () => { | 20 | 39 | throw new Error('should not count as a failure'); | 21 | 40 | }); | 22 | 41 |
| 23 | 42 | it.skip('sync skip pass', () => { | 24 | 43 | }); | 25 | 44 |
| | 45 | +it.skip('sync skip expect fail', { expectFailure: true }, () => { | | 46 | + throw new Error('should not fail'); | | 47 | +}); | | 48 | + | 26 | 49 | it('sync skip pass with message', { skip: 'this is skipped' }, () => { | 27 | 50 | }); | 28 | 51 |
| |
|