GeezerVison is showing github.com — test_runner: add initial TAP parser · nodejs/node@16bedba
File treeExpand file treeCollapse file tree Open diff view settings Expand file treeCollapse file tree Open diff view settings Collapse file Copy file name to clipboardExpand all lines: doc/api/errors.md +19Lines changed: 19 additions & 0 deletions - Display the source diff
- Display the rich diff
| Original file line number | Diff line number | Diff line change |
|---|
@@ -2682,6 +2682,25 @@ An unspecified or non-specific system error has occurred within the Node.js | 2682 | 2682 | process. The error object will have an `err.info` object property with | 2683 | 2683 | additional details. | 2684 | 2684 |
| | 2685 | +<a id="ERR_TAP_LEXER_ERROR"></a> | | 2686 | + | | 2687 | +### `ERR_TAP_LEXER_ERROR` | | 2688 | + | | 2689 | +An error representing a failing lexer state. | | 2690 | + | | 2691 | +<a id="ERR_TAP_PARSER_ERROR"></a> | | 2692 | + | | 2693 | +### `ERR_TAP_PARSER_ERROR` | | 2694 | + | | 2695 | +An error representing a failing parser state. Additional information about | | 2696 | +the token causing the error is available via the `cause` property. | | 2697 | + | | 2698 | +<a id="ERR_TAP_VALIDATION_ERROR"></a> | | 2699 | + | | 2700 | +### `ERR_TAP_VALIDATION_ERROR` | | 2701 | + | | 2702 | +This error represents a failed TAP validation. | | 2703 | + | 2685 | 2704 | <a id="ERR_TEST_FAILURE"></a> | 2686 | 2705 |
| 2687 | 2706 | ### `ERR_TEST_FAILURE` | |
Collapse file Copy file name to clipboardExpand all lines: doc/api/test.md +2-3Lines changed: 2 additions & 3 deletions - Display the source diff
- Display the rich diff
| Original file line number | Diff line number | Diff line change |
|---|
@@ -1028,8 +1028,7 @@ Emitted when [`context.diagnostic`][] is called. | 1028 | 1028 | ### Event: `'test:fail'` | 1029 | 1029 |
| 1030 | 1030 | * `data` {Object} | 1031 | | - * `duration` {number} The test duration. | 1032 | | - * `error` {Error} The failure casing test to fail. | | 1031 | + * `details` {Object} Additional execution metadata. | 1033 | 1032 | * `name` {string} The test name. | 1034 | 1033 | * `testNumber` {number} The ordinal number of the test. | 1035 | 1034 | * `todo` {string|undefined} Present if [`context.todo`][] is called | @@ -1040,7 +1039,7 @@ Emitted when a test fails. | 1040 | 1039 | ### Event: `'test:pass'` | 1041 | 1040 |
| 1042 | 1041 | * `data` {Object} | 1043 | | - * `duration` {number} The test duration. | | 1042 | + * `details` {Object} Additional execution metadata. | 1044 | 1043 | * `name` {string} The test name. | 1045 | 1044 | * `testNumber` {number} The ordinal number of the test. | 1046 | 1045 | * `todo` {string|undefined} Present if [`context.todo`][] is called | |
Collapse file Copy file name to clipboardExpand all lines: lib/internal/errors.js +15Lines changed: 15 additions & 0 deletions | Original file line number | Diff line number | Diff line change |
|---|
@@ -1597,6 +1597,21 @@ E('ERR_STREAM_WRAP', 'Stream has StringDecoder set or is in objectMode', Error); | 1597 | 1597 | E('ERR_STREAM_WRITE_AFTER_END', 'write after end', Error); | 1598 | 1598 | E('ERR_SYNTHETIC', 'JavaScript Callstack', Error); | 1599 | 1599 | E('ERR_SYSTEM_ERROR', 'A system error occurred', SystemError); | | 1600 | +E('ERR_TAP_LEXER_ERROR', function(errorMsg) { | | 1601 | + hideInternalStackFrames(this); | | 1602 | + return errorMsg; | | 1603 | +}, Error); | | 1604 | +E('ERR_TAP_PARSER_ERROR', function(errorMsg, details, tokenCausedError, source) { | | 1605 | + hideInternalStackFrames(this); | | 1606 | + this.cause = tokenCausedError; | | 1607 | + const { column, line, start, end } = tokenCausedError.location; | | 1608 | + const errorDetails = `${details} at line ${line}, column ${column} (start ${start}, end ${end})`; | | 1609 | + return errorMsg + errorDetails; | | 1610 | +}, SyntaxError); | | 1611 | +E('ERR_TAP_VALIDATION_ERROR', function(errorMsg) { | | 1612 | + hideInternalStackFrames(this); | | 1613 | + return errorMsg; | | 1614 | +}, Error); | 1600 | 1615 | E('ERR_TEST_FAILURE', function(error, failureType) { | 1601 | 1616 | hideInternalStackFrames(this); | 1602 | 1617 | assert(typeof failureType === 'string', | |
Collapse file Copy file name to clipboardExpand all lines: lib/internal/test_runner/runner.js +111-4Lines changed: 111 additions & 4 deletions | Original file line number | Diff line number | Diff line change |
|---|
| 2 | 2 | const { | 3 | 3 | ArrayFrom, | 4 | 4 | ArrayPrototypeFilter, | | 5 | + ArrayPrototypeForEach, | 5 | 6 | ArrayPrototypeIncludes, | 6 | 7 | ArrayPrototypeJoin, | 7 | 8 | ArrayPrototypePush, | @@ -14,6 +15,7 @@ const { | 14 | 15 | SafePromiseAllSettledReturnVoid, | 15 | 16 | SafeMap, | 16 | 17 | SafeSet, | | 18 | + StringPrototypeRepeat, | 17 | 19 | } = primordials; | 18 | 20 |
| 19 | 21 | const { spawn } = require('child_process'); | @@ -31,7 +33,10 @@ const { validateArray, validateBoolean } = require('internal/validators'); | 31 | 33 | const { getInspectPort, isUsingInspector, isInspectorMessage } = require('internal/util/inspector'); | 32 | 34 | const { kEmptyObject } = require('internal/util'); | 33 | 35 | const { createTestTree } = require('internal/test_runner/harness'); | 34 | | -const { kSubtestsFailed, Test } = require('internal/test_runner/test'); | | 36 | +const { kDefaultIndent, kSubtestsFailed, Test } = require('internal/test_runner/test'); | | 37 | +const { TapParser } = require('internal/test_runner/tap_parser'); | | 38 | +const { TokenKind } = require('internal/test_runner/tap_lexer'); | | 39 | + | 35 | 40 | const { | 36 | 41 | isSupportedFileType, | 37 | 42 | doesPathMatchFilter, | @@ -119,11 +124,103 @@ function getRunArgs({ path, inspectPort }) { | 119 | 124 | return argv; | 120 | 125 | } | 121 | 126 |
| | 127 | +class FileTest extends Test { | | 128 | + #buffer = []; | | 129 | + #handleReportItem({ kind, node, nesting = 0 }) { | | 130 | + const indent = StringPrototypeRepeat(kDefaultIndent, nesting + 1); | | 131 | + | | 132 | + const details = (diagnostic) => { | | 133 | + return ( | | 134 | + diagnostic && { | | 135 | + __proto__: null, | | 136 | + yaml: | | 137 | + `${indent} ` + | | 138 | + ArrayPrototypeJoin(diagnostic, `\n${indent} `) + | | 139 | + '\n', | | 140 | + } | | 141 | + ); | | 142 | + }; | | 143 | + | | 144 | + switch (kind) { | | 145 | + case TokenKind.TAP_VERSION: | | 146 | + // TODO(manekinekko): handle TAP version coming from the parser. | | 147 | + // this.reporter.version(node.version); | | 148 | + break; | | 149 | + | | 150 | + case TokenKind.TAP_PLAN: | | 151 | + this.reporter.plan(indent, node.end - node.start + 1); | | 152 | + break; | | 153 | + | | 154 | + case TokenKind.TAP_SUBTEST_POINT: | | 155 | + this.reporter.subtest(indent, node.name); | | 156 | + break; | | 157 | + | | 158 | + case TokenKind.TAP_TEST_POINT: | | 159 | + // eslint-disable-next-line no-case-declarations | | 160 | + const { todo, skip, pass } = node.status; | | 161 | + // eslint-disable-next-line no-case-declarations | | 162 | + let directive; | | 163 | + | | 164 | + if (skip) { | | 165 | + directive = this.reporter.getSkip(node.reason); | | 166 | + } else if (todo) { | | 167 | + directive = this.reporter.getTodo(node.reason); | | 168 | + } else { | | 169 | + directive = kEmptyObject; | | 170 | + } | | 171 | + | | 172 | + if (pass) { | | 173 | + this.reporter.ok( | | 174 | + indent, | | 175 | + node.id, | | 176 | + node.description, | | 177 | + details(node.diagnostics), | | 178 | + directive | | 179 | + ); | | 180 | + } else { | | 181 | + this.reporter.fail( | | 182 | + indent, | | 183 | + node.id, | | 184 | + node.description, | | 185 | + details(node.diagnostics), | | 186 | + directive | | 187 | + ); | | 188 | + } | | 189 | + break; | | 190 | + | | 191 | + case TokenKind.COMMENT: | | 192 | + if (indent === kDefaultIndent) { | | 193 | + // Ignore file top level diagnostics | | 194 | + break; | | 195 | + } | | 196 | + this.reporter.diagnostic(indent, node.comment); | | 197 | + break; | | 198 | + | | 199 | + case TokenKind.UNKNOWN: | | 200 | + this.reporter.diagnostic(indent, node.value); | | 201 | + break; | | 202 | + } | | 203 | + } | | 204 | + addToReport(ast) { | | 205 | + if (!this.isClearToSend()) { | | 206 | + ArrayPrototypePush(this.#buffer, ast); | | 207 | + return; | | 208 | + } | | 209 | + this.reportSubtest(); | | 210 | + this.#handleReportItem(ast); | | 211 | + } | | 212 | + report() { | | 213 | + this.reportSubtest(); | | 214 | + ArrayPrototypeForEach(this.#buffer, (ast) => this.#handleReportItem(ast)); | | 215 | + super.report(); | | 216 | + } | | 217 | +} | | 218 | + | 122 | 219 | const runningProcesses = new SafeMap(); | 123 | 220 | const runningSubtests = new SafeMap(); | 124 | 221 |
| 125 | 222 | function runTestFile(path, root, inspectPort, filesWatcher) { | 126 | | - const subtest = root.createSubtest(Test, path, async (t) => { | | 223 | + const subtest = root.createSubtest(FileTest, path, async (t) => { | 127 | 224 | const args = getRunArgs({ path, inspectPort }); | 128 | 225 | const stdio = ['pipe', 'pipe', 'pipe']; | 129 | 226 | const env = { ...process.env }; | @@ -134,8 +231,7 @@ function runTestFile(path, root, inspectPort, filesWatcher) { | 134 | 231 |
| 135 | 232 | const child = spawn(process.execPath, args, { signal: t.signal, encoding: 'utf8', env, stdio }); | 136 | 233 | runningProcesses.set(path, child); | 137 | | - // TODO(cjihrig): Implement a TAP parser to read the child's stdout | 138 | | - // instead of just displaying it all if the child fails. | | 234 | + | 139 | 235 | let err; | 140 | 236 | let stderr = ''; | 141 | 237 |
| @@ -158,6 +254,17 @@ function runTestFile(path, root, inspectPort, filesWatcher) { | 158 | 254 | }); | 159 | 255 | } | 160 | 256 |
| | 257 | + const parser = new TapParser(); | | 258 | + child.stderr.pipe(parser).on('data', (ast) => { | | 259 | + if (ast.lexeme && isInspectorMessage(ast.lexeme)) { | | 260 | + process.stderr.write(ast.lexeme + '\n'); | | 261 | + } | | 262 | + }); | | 263 | + | | 264 | + child.stdout.pipe(parser).on('data', (ast) => { | | 265 | + subtest.addToReport(ast); | | 266 | + }); | | 267 | + | 161 | 268 | const { 0: { 0: code, 1: signal }, 1: stdout } = await SafePromiseAll([ | 162 | 269 | once(child, 'exit', { signal: t.signal }), | 163 | 270 | child.stdout.toArray({ signal: t.signal }), | |
Collapse file Copy file name to clipboard +155Lines changed: 155 additions & 0 deletions | Original file line number | Diff line number | Diff line change |
|---|
| | 1 | +'use strict'; | | 2 | + | | 3 | +const { | | 4 | + ArrayPrototypeFilter, | | 5 | + ArrayPrototypeFind, | | 6 | + NumberParseInt, | | 7 | +} = primordials; | | 8 | +const { | | 9 | + codes: { ERR_TAP_VALIDATION_ERROR }, | | 10 | +} = require('internal/errors'); | | 11 | +const { TokenKind } = require('internal/test_runner/tap_lexer'); | | 12 | + | | 13 | +// TODO(@manekinekko): add more validation rules based on the TAP14 spec. | | 14 | +// See https://testanything.org/tap-version-14-specification.html | | 15 | +class TAPValidationStrategy { | | 16 | + validate(ast) { | | 17 | + this.#validateVersion(ast); | | 18 | + this.#validatePlan(ast); | | 19 | + this.#validateTestPoints(ast); | | 20 | + | | 21 | + return true; | | 22 | + } | | 23 | + | | 24 | + #validateVersion(ast) { | | 25 | + const entry = ArrayPrototypeFind( | | 26 | + ast, | | 27 | + (node) => node.kind === TokenKind.TAP_VERSION | | 28 | + ); | | 29 | + | | 30 | + if (!entry) { | | 31 | + throw new ERR_TAP_VALIDATION_ERROR('missing TAP version'); | | 32 | + } | | 33 | + | | 34 | + const { version } = entry.node; | | 35 | + | | 36 | + // TAP14 specification is compatible with observed behavior of existing TAP13 consumers and producers | | 37 | + if (version !== '14' && version !== '13') { | | 38 | + throw new ERR_TAP_VALIDATION_ERROR('TAP version should be 13 or 14'); | | 39 | + } | | 40 | + } | | 41 | + | | 42 | + #validatePlan(ast) { | | 43 | + const entry = ArrayPrototypeFind( | | 44 | + ast, | | 45 | + (node) => node.kind === TokenKind.TAP_PLAN | | 46 | + ); | | 47 | + | | 48 | + if (!entry) { | | 49 | + throw new ERR_TAP_VALIDATION_ERROR('missing TAP plan'); | | 50 | + } | | 51 | + | | 52 | + const plan = entry.node; | | 53 | + | | 54 | + if (!plan.start) { | | 55 | + throw new ERR_TAP_VALIDATION_ERROR('missing plan start'); | | 56 | + } | | 57 | + | | 58 | + if (!plan.end) { | | 59 | + throw new ERR_TAP_VALIDATION_ERROR('missing plan end'); | | 60 | + } | | 61 | + | | 62 | + const planStart = NumberParseInt(plan.start, 10); | | 63 | + const planEnd = NumberParseInt(plan.end, 10); | | 64 | + | | 65 | + if (planEnd !== 0 && planStart > planEnd) { | | 66 | + throw new ERR_TAP_VALIDATION_ERROR( | | 67 | + `plan start ${planStart} is greater than plan end ${planEnd}` | | 68 | + ); | | 69 | + } | | 70 | + } | | 71 | + | | 72 | + // TODO(@manekinekko): since we are dealing with a flat AST, we need to | | 73 | + // validate test points grouped by their "nesting" level. This is because a set of | | 74 | + // Test points belongs to a TAP document. Each new subtest block creates a new TAP document. | | 75 | + // https://testanything.org/tap-version-14-specification.html#subtests | | 76 | + #validateTestPoints(ast) { | | 77 | + const bailoutEntry = ArrayPrototypeFind( | | 78 | + ast, | | 79 | + (node) => node.kind === TokenKind.TAP_BAIL_OUT | | 80 | + ); | | 81 | + const planEntry = ArrayPrototypeFind( | | 82 | + ast, | | 83 | + (node) => node.kind === TokenKind.TAP_PLAN | | 84 | + ); | | 85 | + const testPointEntries = ArrayPrototypeFilter( | | 86 | + ast, | | 87 | + (node) => node.kind === TokenKind.TAP_TEST_POINT | | 88 | + ); | | 89 | + | | 90 | + const plan = planEntry.node; | | 91 | + | | 92 | + const planStart = NumberParseInt(plan.start, 10); | | 93 | + const planEnd = NumberParseInt(plan.end, 10); | | 94 | + | | 95 | + if (planEnd === 0 && testPointEntries.length > 0) { | | 96 | + throw new ERR_TAP_VALIDATION_ERROR( | | 97 | + `found ${testPointEntries.length} Test Point${ | | 98 | + testPointEntries.length > 1 ? 's' : '' | | 99 | + } but plan is ${planStart}..0` | | 100 | + ); | | 101 | + } | | 102 | + | | 103 | + if (planEnd > 0) { | | 104 | + if (testPointEntries.length === 0) { | | 105 | + throw new ERR_TAP_VALIDATION_ERROR('missing Test Points'); | | 106 | + } | | 107 | + | | 108 | + if (!bailoutEntry && testPointEntries.length !== planEnd) { | | 109 | + throw new ERR_TAP_VALIDATION_ERROR( | | 110 | + `test Points count ${testPointEntries.length} does not match plan count ${planEnd}` | | 111 | + ); | | 112 | + } | | 113 | + | | 114 | + for (let i = 0; i < testPointEntries.length; i++) { | | 115 | + const test = testPointEntries[i].node; | | 116 | + const testId = NumberParseInt(test.id, 10); | | 117 | + | | 118 | + if (testId < planStart || testId > planEnd) { | | 119 | + throw new ERR_TAP_VALIDATION_ERROR( | | 120 | + `test ${testId} is out of plan range ${planStart}..${planEnd}` | | 121 | + ); | | 122 | + } | | 123 | + } | | 124 | + } | | 125 | + } | | 126 | +} | | 127 | + | | 128 | +// TAP14 and TAP13 are compatible with each other | | 129 | +class TAP13ValidationStrategy extends TAPValidationStrategy {} | | 130 | +class TAP14ValidationStrategy extends TAPValidationStrategy {} | | 131 | + | | 132 | +class TapChecker { | | 133 | + static TAP13 = '13'; | | 134 | + static TAP14 = '14'; | | 135 | + | | 136 | + constructor({ specs }) { | | 137 | + switch (specs) { | | 138 | + case TapChecker.TAP13: | | 139 | + this.strategy = new TAP13ValidationStrategy(); | | 140 | + break; | | 141 | + default: | | 142 | + this.strategy = new TAP14ValidationStrategy(); | | 143 | + } | | 144 | + } | | 145 | + | | 146 | + check(ast) { | | 147 | + return this.strategy.validate(ast); | | 148 | + } | | 149 | +} | | 150 | + | | 151 | +module.exports = { | | 152 | + TapChecker, | | 153 | + TAP14ValidationStrategy, | | 154 | + TAP13ValidationStrategy, | | 155 | +}; |
|