@@ -58,46 +58,66 @@ function codeUnitStr(char) { |
58 | 58 | return 'U+' + char.charCodeAt(0).toString(16); |
59 | 59 | } |
60 | 60 |
|
| 61 | +class ReportResult { |
| 62 | + constructor(name) { |
| 63 | + this.test = name; |
| 64 | + this.status = 'OK'; |
| 65 | + this.subtests = []; |
| 66 | + } |
| 67 | + |
| 68 | + addSubtest(name, status, message) { |
| 69 | + const subtest = { |
| 70 | + status, |
| 71 | + // https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3722 |
| 72 | + name: sanitizeUnpairedSurrogates(name), |
| 73 | + }; |
| 74 | + if (message) { |
| 75 | + // https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L4506 |
| 76 | + subtest.message = sanitizeUnpairedSurrogates(message); |
| 77 | + } |
| 78 | + this.subtests.push(subtest); |
| 79 | + return subtest; |
| 80 | + } |
| 81 | + |
| 82 | + finish(status) { |
| 83 | + this.status = status ?? 'OK'; |
| 84 | + } |
| 85 | +} |
| 86 | + |
| 87 | +// Generates a report that can be uploaded to wpt.fyi. |
| 88 | +// Checkout https://github.com/web-platform-tests/wpt.fyi/tree/main/api#results-creation |
| 89 | +// for more details. |
61 | 90 | class WPTReport { |
62 | 91 | constructor(path) { |
63 | 92 | this.filename = `report-${path.replaceAll('/', '-')}.json`; |
64 | | - this.results = []; |
| 93 | + /** @type {Map<string, ReportResult>} */ |
| 94 | + this.results = new Map(); |
65 | 95 | this.time_start = Date.now(); |
66 | 96 | } |
67 | 97 |
|
68 | | - addResult(name, status) { |
69 | | - const result = { |
70 | | - test: name, |
71 | | - status, |
72 | | - subtests: [], |
73 | | - addSubtest(name, status, message) { |
74 | | - const subtest = { |
75 | | - status, |
76 | | - // https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3722 |
77 | | - name: sanitizeUnpairedSurrogates(name), |
78 | | - }; |
79 | | - if (message) { |
80 | | - // https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L4506 |
81 | | - subtest.message = sanitizeUnpairedSurrogates(message); |
82 | | - } |
83 | | - this.subtests.push(subtest); |
84 | | - return subtest; |
85 | | - }, |
86 | | - }; |
87 | | - this.results.push(result); |
| 98 | + /** |
| 99 | + * Get or create a ReportResult for a test spec. |
| 100 | + * @param {WPTTestSpec} spec |
| 101 | + */ |
| 102 | + getResult(spec) { |
| 103 | + const name = `/${spec.getRelativePath()}${spec.variant}`; |
| 104 | + if (this.results.has(name)) { |
| 105 | + return this.results.get(name); |
| 106 | + } |
| 107 | + const result = new ReportResult(name); |
| 108 | + this.results.set(name, result); |
88 | 109 | return result; |
89 | 110 | } |
90 | 111 |
|
91 | 112 | write() { |
92 | 113 | this.time_end = Date.now(); |
93 | | - this.results = this.results.filter((result) => { |
94 | | - return result.status === 'SKIP' || result.subtests.length !== 0; |
95 | | - }).map((result) => { |
96 | | - const url = new URL(result.test, 'http://wpt'); |
97 | | - url.pathname = url.pathname.replace(/\.js$/, '.html'); |
98 | | - result.test = url.href.slice(url.origin.length); |
99 | | - return result; |
100 | | - }); |
| 114 | + const results = Array.from(this.results.values()) |
| 115 | + .map((result) => { |
| 116 | + const url = new URL(result.test, 'http://wpt'); |
| 117 | + url.pathname = url.pathname.replace(/\.js$/, '.html'); |
| 118 | + result.test = url.href.slice(url.origin.length); |
| 119 | + return result; |
| 120 | + }); |
101 | 121 |
|
102 | 122 | /** |
103 | 123 | * Return required and some optional properties |
@@ -110,7 +130,12 @@ class WPTReport { |
110 | 130 | os: getOs(), |
111 | 131 | }; |
112 | 132 |
|
113 | | - fs.writeFileSync(`out/wpt/${this.filename}`, JSON.stringify(this)); |
| 133 | + fs.writeFileSync(`out/wpt/${this.filename}`, JSON.stringify({ |
| 134 | + time_start: this.time_start, |
| 135 | + time_end: this.time_end, |
| 136 | + run_info: this.run_info, |
| 137 | + results: results, |
| 138 | + })); |
114 | 139 | } |
115 | 140 | } |
116 | 141 |
|
@@ -642,14 +667,13 @@ class WPTRunner { |
642 | 667 | this.inProgress.add(spec); |
643 | 668 | this.workers.set(spec, worker); |
644 | 669 |
|
645 | | - let reportResult; |
| 670 | + const reportResult = this.report?.getResult(spec); |
646 | 671 | worker.on('message', (message) => { |
647 | 672 | switch (message.type) { |
648 | 673 | case 'result': |
649 | | - reportResult ||= this.report?.addResult(`/${relativePath}${spec.variant}`, 'OK'); |
650 | 674 | return this.resultCallback(spec, message.result, reportResult); |
651 | 675 | case 'completion': |
652 | | - return this.completionCallback(spec, message.status); |
| 676 | + return this.completionCallback(spec, message.status, reportResult); |
653 | 677 | default: |
654 | 678 | throw new Error(`Unexpected message from worker: ${message.type}`); |
655 | 679 | } |
@@ -661,6 +685,8 @@ class WPTRunner { |
661 | 685 | // This can happen normally, for example in timers tests. |
662 | 686 | return; |
663 | 687 | } |
| 688 | + // Generate a subtest failure for visibility. |
| 689 | + // No need to record this synthetic failure with wpt.fyi. |
664 | 690 | this.fail( |
665 | 691 | spec, |
666 | 692 | { |
@@ -671,6 +697,8 @@ class WPTRunner { |
671 | 697 | }, |
672 | 698 | kUncaught, |
673 | 699 | ); |
| 700 | + // Mark the whole test as failed in wpt.fyi report. |
| 701 | + reportResult?.finish('ERROR'); |
674 | 702 | this.inProgress.delete(spec); |
675 | 703 | }); |
676 | 704 |
|
@@ -680,7 +708,11 @@ class WPTRunner { |
680 | 708 |
|
681 | 709 | process.on('exit', () => { |
682 | 710 | for (const spec of this.inProgress) { |
| 711 | + // No need to record this synthetic failure with wpt.fyi. |
683 | 712 | this.fail(spec, { name: 'Incomplete' }, kIncomplete); |
| 713 | + // Mark the whole test as failed in wpt.fyi report. |
| 714 | + const reportResult = this.report?.getResult(spec); |
| 715 | + reportResult?.finish('ERROR'); |
684 | 716 | } |
685 | 717 | inspect.defaultOptions.depth = Infinity; |
686 | 718 | // Sorts the rules to have consistent output |
@@ -780,6 +812,7 @@ class WPTRunner { |
780 | 812 | * in one test file). |
781 | 813 | * @param {WPTTestSpec} spec |
782 | 814 | * @param {Test} test The Test object returned by WPT harness |
| 815 | + * @param {ReportResult} reportResult The report result object |
783 | 816 | */ |
784 | 817 | resultCallback(spec, test, reportResult) { |
785 | 818 | const status = this.getTestStatus(test.status); |
@@ -794,13 +827,19 @@ class WPTRunner { |
794 | 827 | * Report the status of each WPT test (one per file) |
795 | 828 | * @param {WPTTestSpec} spec |
796 | 829 | * @param {object} harnessStatus - The status object returned by WPT harness. |
| 830 | + * @param {ReportResult} reportResult The report result object |
797 | 831 | */ |
798 | | - completionCallback(spec, harnessStatus) { |
| 832 | + completionCallback(spec, harnessStatus, reportResult) { |
799 | 833 | const status = this.getTestStatus(harnessStatus.status); |
800 | 834 |
|
801 | 835 | // Treat it like a test case failure |
802 | 836 | if (status === kTimeout) { |
| 837 | + // No need to record this synthetic failure with wpt.fyi. |
803 | 838 | this.fail(spec, { name: 'WPT testharness timeout' }, kTimeout); |
| 839 | + // Mark the whole test as TIMEOUT in wpt.fyi report. |
| 840 | + reportResult?.finish('TIMEOUT'); |
| 841 | + } else { |
| 842 | + reportResult?.finish(); |
804 | 843 | } |
805 | 844 | this.inProgress.delete(spec); |
806 | 845 | // Always force termination of the worker. Some tests allocate resources |
|