GV

GeezerVison

The World Wide Web, restored to 800 pixels of glory.

Home|About|Gaming cabinet|Archives|How it works|Starter sites

On this site

  • Sign in
  • ActionsCI/CD and automation
  • CodespacesCloud development environments
  • All Features
  • Documentation
  • Training
  • Certifications
  • Build with Copilot
  • Customize Copilot
  • Integrations
  • Trending repos
  • GitHub Sponsors
  • Developer articles
  • Enterprises

This view

  • Original page
  • New address

GeezerVison is showing github.com — test: run WPT files in parallel again · nodejs/node@54261f3

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
  • Makefile
  • test
    • common
      • wpt.js
    • wpt
      • testcfg.py
  • tools
    • merge-wpt-reports.mjs
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎Makefile‎

Copy file name to clipboardExpand all lines: Makefile
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -603,6 +603,7 @@ test-wpt-report:
603603
$(RM) -r out/wpt
604604
mkdir -p out/wpt
605605
WPT_REPORT=1 $(PYTHON) tools/test.py --shell $(NODE) $(PARALLEL_ARGS) wpt
606+
$(NODE) "$$PWD/tools/merge-wpt-reports.mjs"
606607

607608
.PHONY: test-simple
608609
test-simple: | cctest # Depends on 'all'.
Collapse file

‎test/common/wpt.js‎

Copy file name to clipboardExpand all lines: test/common/wpt.js
+53-30Lines changed: 53 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ const os = require('os');
1010
const { inspect } = require('util');
1111
const { Worker } = require('worker_threads');
1212

13+
const workerPath = path.join(__dirname, 'wpt/worker.js');
14+
1315
function getBrowserProperties() {
1416
const { node: version } = process.versions; // e.g. 18.13.0, 20.0.0-nightly202302078e6e215481
1517
const release = /^\d+\.\d+\.\d+$/.test(version);
@@ -57,7 +59,8 @@ function codeUnitStr(char) {
5759
}
5860

5961
class WPTReport {
60-
constructor() {
62+
constructor(path) {
63+
this.filename = `report-${path.replaceAll('/', '-')}.json`;
6164
this.results = [];
6265
this.time_start = Date.now();
6366
}
@@ -96,26 +99,18 @@ class WPTReport {
9699
return result;
97100
});
98101

99-
if (fs.existsSync('out/wpt/wptreport.json')) {
100-
const prev = JSON.parse(fs.readFileSync('out/wpt/wptreport.json'));
101-
this.results = [...prev.results, ...this.results];
102-
this.time_start = prev.time_start;
103-
this.time_end = Math.max(this.time_end, prev.time_end);
104-
this.run_info = prev.run_info;
105-
} else {
106-
/**
107-
* Return required and some optional properties
108-
* https://github.com/web-platform-tests/wpt.fyi/blob/60da175/api/README.md?plain=1#L331-L335
109-
*/
110-
this.run_info = {
111-
product: 'node.js',
112-
...getBrowserProperties(),
113-
revision: process.env.WPT_REVISION || 'unknown',
114-
os: getOs(),
115-
};
116-
}
102+
/**
103+
* Return required and some optional properties
104+
* https://github.com/web-platform-tests/wpt.fyi/blob/60da175/api/README.md?plain=1#L331-L335
105+
*/
106+
this.run_info = {
107+
product: 'node.js',
108+
...getBrowserProperties(),
109+
revision: process.env.WPT_REVISION || 'unknown',
110+
os: getOs(),
111+
};
117112

118-
fs.writeFileSync('out/wpt/wptreport.json', JSON.stringify(this));
113+
fs.writeFileSync(`out/wpt/${this.filename}`, JSON.stringify(this));
119114
}
120115
}
121116

@@ -402,6 +397,29 @@ const kIncomplete = 'incomplete';
402397
const kUncaught = 'uncaught';
403398
const NODE_UNCAUGHT = 100;
404399

400+
const limit = (concurrency) => {
401+
let running = 0;
402+
const queue = [];
403+
404+
const execute = async (fn) => {
405+
if (running < concurrency) {
406+
running++;
407+
try {
408+
await fn();
409+
} finally {
410+
running--;
411+
if (queue.length > 0) {
412+
execute(queue.shift());
413+
}
414+
}
415+
} else {
416+
queue.push(fn);
417+
}
418+
};
419+
420+
return execute;
421+
};
422+
405423
class WPTRunner {
406424
constructor(path) {
407425
this.path = path;
@@ -425,7 +443,7 @@ class WPTRunner {
425443
this.scriptsModifier = null;
426444

427445
if (process.env.WPT_REPORT != null) {
428-
this.report = new WPTReport();
446+
this.report = new WPTReport(path);
429447
}
430448
}
431449

@@ -543,6 +561,8 @@ class WPTRunner {
543561

544562
this.inProgress = new Set(queue.map((spec) => spec.filename));
545563

564+
const run = limit(os.availableParallelism());
565+
546566
for (const spec of queue) {
547567
const testFileName = spec.filename;
548568
const content = spec.getContent();
@@ -576,15 +596,7 @@ class WPTRunner {
576596
this.scriptsModifier?.(obj);
577597
scriptsToRun.push(obj);
578598

579-
/**
580-
* Example test with no META variant
581-
* https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/sign_verify/hmac.https.any.js#L1-L4
582-
*
583-
* Example test with multiple META variants
584-
* https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/generateKey/successes_RSASSA-PKCS1-v1_5.https.any.js#L1-L9
585-
*/
586-
for (const variant of meta.variant || ['']) {
587-
const workerPath = path.join(__dirname, 'wpt/worker.js');
599+
const runWorker = async (variant) => {
588600
const worker = new Worker(workerPath, {
589601
execArgv: this.flags,
590602
workerData: {
@@ -635,6 +647,17 @@ class WPTRunner {
635647
});
636648

637649
await events.once(worker, 'exit').catch(() => {});
650+
};
651+
652+
/**
653+
* Example test with no META variant
654+
* https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/sign_verify/hmac.https.any.js#L1-L4
655+
*
656+
* Example test with multiple META variants
657+
* https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/generateKey/successes_RSASSA-PKCS1-v1_5.https.any.js#L1-L9
658+
*/
659+
for (const variant of meta.variant || ['']) {
660+
run(() => runWorker(variant));
638661
}
639662
}
640663

Collapse file

‎test/wpt/testcfg.py‎

Copy file name to clipboardExpand all lines: test/wpt/testcfg.py
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@
33
import testpy
44

55
def GetConfiguration(context, root):
6-
return testpy.SimpleTestConfiguration(context, root, 'wpt')
6+
return testpy.ParallelTestConfiguration(context, root, 'wpt')
Collapse file

‎tools/merge-wpt-reports.mjs‎

Copy file name to clipboard
+32Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import * as fs from 'node:fs';
2+
import * as path from 'node:path';
3+
import * as url from 'node:url';
4+
5+
const __dirname = path.dirname(url.fileURLToPath(import.meta.url));
6+
7+
const outDir = path.resolve(__dirname, '../out/wpt');
8+
const files = fs.readdirSync(outDir);
9+
const reports = files.filter((file) => file.endsWith('.json'));
10+
11+
const startTimes = [];
12+
const endTimes = [];
13+
const results = [];
14+
let runInfo;
15+
16+
for (const file of reports) {
17+
const report = JSON.parse(fs.readFileSync(path.resolve(outDir, file)));
18+
fs.unlinkSync(path.resolve(outDir, file));
19+
results.push(...report.results);
20+
startTimes.push(report.time_start);
21+
endTimes.push(report.time_end);
22+
runInfo ||= report.run_info;
23+
}
24+
25+
const mergedReport = {
26+
time_start: Math.min(...startTimes),
27+
time_end: Math.max(...endTimes),
28+
run_info: runInfo,
29+
results,
30+
};
31+
32+
fs.writeFileSync(path.resolve(outDir, 'wptreport.json'), JSON.stringify(mergedReport));

0 commit comments

Comments
0 (0)
You are visitor 01402503 — GeezerVison does not store the pages you look up.
Best viewed with Netscape Navigator 4.7 or Internet Explorer 6 at 800×600. No cookies required.