GV

GeezerVison

The World Wide Web, restored to 800 pixels of glory.

Home|About|Gaming cabinet|Archives|How it works|Starter sites

On this site

  • Sign in
  • ActionsCI/CD and automation
  • CodespacesCloud development environments
  • All Features
  • Documentation
  • Training
  • Certifications
  • Build with Copilot
  • Customize Copilot
  • Integrations
  • Trending repos
  • GitHub Sponsors
  • Developer articles
  • Enterprises

This view

  • Original page
  • New address

GeezerVison is showing github.com — fs: use sized reads for large files in readFileUtf8 · nodejs/node@e374abd

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
  • src
    • node_file.cc
  • test/parallel
    • test-fs-readfilesync-utf8-sizes.js
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎src/node_file.cc‎

Copy file name to clipboardExpand all lines: src/node_file.cc
+62-2Lines changed: 62 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2923,10 +2923,20 @@ static void ReadFileUtf8(const FunctionCallbackInfo<Value>& args) {
29232923
uv_fs_req_cleanup(&req);
29242924
});
29252925

2926+
// Past the first 8 KiB, read into one heap buffer sized from fstat(); the
2927+
// size is only a hint, reading continues until read() reports EOF.
29262928
std::string result{};
29272929
char buffer[8192];
29282930
uv_buf_t buf = uv_buf_init(buffer, sizeof(buffer));
29292931

2932+
char* big = nullptr;
2933+
size_t big_len = 0;
2934+
size_t big_cap = 0;
2935+
bool sized = false;
2936+
auto free_big = OnScopeLeave([&big]() { free(big); });
2937+
constexpr size_t kMinChunk = 64 * 1024;
2938+
constexpr size_t kMaxChunk = 8 * 1024 * 1024;
2939+
29302940
FS_SYNC_TRACE_BEGIN(read);
29312941
while (true) {
29322942
auto r = uv_fs_read(nullptr, &req, file, &buf, 1, -1, nullptr);
@@ -2939,12 +2949,62 @@ static void ReadFileUtf8(const FunctionCallbackInfo<Value>& args) {
29392949
if (r <= 0) {
29402950
break;
29412951
}
2942-
result.append(buf.base, r);
2952+
if (big == nullptr) {
2953+
result.append(buf.base, r);
2954+
if (static_cast<size_t>(r) < sizeof(buffer)) {
2955+
continue;
2956+
}
2957+
// Switch to the heap buffer.
2958+
uv_fs_req_cleanup(&req);
2959+
big_cap = kMinChunk;
2960+
big = UncheckedMalloc<char>(big_cap);
2961+
if (big == nullptr) {
2962+
FS_SYNC_TRACE_END(read);
2963+
return THROW_ERR_MEMORY_ALLOCATION_FAILED(env);
2964+
}
2965+
memcpy(big, result.data(), result.size());
2966+
big_len = result.size();
2967+
result = std::string();
2968+
} else {
2969+
big_len += static_cast<size_t>(r);
2970+
}
2971+
if (big_len == big_cap) {
2972+
// +1 leaves room for the read() that reports EOF.
2973+
size_t new_cap =
2974+
big_cap + std::min(kMaxChunk, std::max(kMinChunk, big_cap));
2975+
if (!sized) {
2976+
sized = true;
2977+
uv_fs_req_cleanup(&req);
2978+
uv_fs_t stat_req;
2979+
if (uv_fs_fstat(nullptr, &stat_req, file, nullptr) == 0) {
2980+
const uv_stat_t* const st =
2981+
static_cast<const uv_stat_t*>(stat_req.ptr);
2982+
if ((st->st_mode & S_IFMT) == S_IFREG &&
2983+
static_cast<uint64_t>(st->st_size) > big_len &&
2984+
static_cast<uint64_t>(st->st_size) <
2985+
static_cast<uint64_t>(v8::String::kMaxLength)) {
2986+
new_cap = static_cast<size_t>(st->st_size) + 1;
2987+
}
2988+
}
2989+
uv_fs_req_cleanup(&stat_req);
2990+
}
2991+
char* const grown = UncheckedRealloc<char>(big, new_cap);
2992+
if (grown == nullptr) {
2993+
FS_SYNC_TRACE_END(read);
2994+
return THROW_ERR_MEMORY_ALLOCATION_FAILED(env);
2995+
}
2996+
big = grown;
2997+
big_cap = new_cap;
2998+
}
2999+
buf = uv_buf_init(big + big_len, std::min(kMaxChunk, big_cap - big_len));
29433000
}
29443001
FS_SYNC_TRACE_END(read);
29453002

29463003
Local<Value> val;
2947-
if (!ToV8Value(env->context(), result, isolate).ToLocal(&val)) {
3004+
const std::string_view content = big != nullptr
3005+
? std::string_view(big, big_len)
3006+
: std::string_view(result);
3007+
if (!ToV8Value(env->context(), content, isolate).ToLocal(&val)) {
29483008
return;
29493009
}
29503010

Collapse file

‎test/parallel/test-fs-readfilesync-utf8-sizes.js‎

Copy file name to clipboard
+98Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
'use strict';
2+
// fs.readFileSync(path, 'utf8') takes a dedicated native path. Its result must
3+
// equal fs.readFileSync(path).toString('utf8') for every file size (in
4+
// particular around its internal 8 KiB stack buffer and for multi-megabyte
5+
// files), for file descriptors positioned mid-file, and for files whose
6+
// reported size is wrong (procfs reports 0, sysfs reports a page).
7+
const common = require('../common');
8+
const tmpdir = require('../common/tmpdir');
9+
const assert = require('assert');
10+
const fs = require('fs');
11+
12+
tmpdir.refresh();
13+
14+
function content(size) {
15+
// Multi-byte characters straddling every possible chunk boundary.
16+
const unit = 'abcdé€\u{1F600}\n';
17+
let s = unit.repeat(Math.ceil(size / unit.length));
18+
s = s.slice(0, size);
19+
// Avoid ending on a lone surrogate produced by slice().
20+
if (/[\ud800-\udbff]$/.test(s)) s = s.slice(0, -1) + 'x';
21+
return s;
22+
}
23+
24+
const sizes = [0, 1, 8190, 8191, 8192, 8193, 8194, 16383, 16384, 16385,
25+
65535, 65536, 65537, 100000, (1 << 20) - 1, 1 << 20, (1 << 20) + 1,
26+
(8 << 20) + 5];
27+
for (const size of sizes) {
28+
const file = tmpdir.resolve(`f-${size}.txt`);
29+
const str = content(size);
30+
fs.writeFileSync(file, str);
31+
const expected = fs.readFileSync(file).toString('utf8');
32+
assert.strictEqual(fs.readFileSync(file, 'utf8'), expected, `size ${size} by path`);
33+
assert.strictEqual(fs.readFileSync(file, { encoding: 'utf-8' }), expected, `size ${size} utf-8 alias`);
34+
// By fd: from the start (leaves the fd at EOF), then at EOF, then from a
35+
// mid-file position on a fresh fd.
36+
let fd = fs.openSync(file, 'r');
37+
try {
38+
assert.strictEqual(fs.readFileSync(fd, 'utf8'), expected, `size ${size} by fd`);
39+
assert.strictEqual(fs.readFileSync(fd, 'utf8'), '', `size ${size} by fd at EOF`);
40+
} finally {
41+
fs.closeSync(fd);
42+
}
43+
if (size > 10) {
44+
fd = fs.openSync(file, 'r');
45+
try {
46+
// Advance the fd 3 bytes (inside the ASCII prefix, so still valid UTF-8).
47+
assert.strictEqual(fs.readSync(fd, Buffer.alloc(3), 0, 3, null), 3);
48+
assert.strictEqual(fs.readFileSync(fd, 'utf8'), Buffer.from(expected).subarray(3).toString('utf8'),
49+
`size ${size} by fd at offset 3`);
50+
} finally {
51+
fs.closeSync(fd);
52+
}
53+
}
54+
}
55+
56+
// Binary garbage is decoded with replacement characters identically.
57+
{
58+
const file = tmpdir.resolve('binary.bin');
59+
const buf = Buffer.alloc(20000);
60+
for (let i = 0; i < buf.length; i++) buf[i] = (i * 7919) & 0xff;
61+
fs.writeFileSync(file, buf);
62+
assert.strictEqual(fs.readFileSync(file, 'utf8'), buf.toString('utf8'));
63+
}
64+
65+
// Files whose st_size does not describe their content.
66+
if (common.isLinux) {
67+
for (const file of ['/proc/self/status', '/proc/self/smaps', '/proc/cpuinfo',
68+
'/proc/version', '/sys/kernel/mm/transparent_hugepage/enabled']) {
69+
let viaBuffer;
70+
try {
71+
viaBuffer = fs.readFileSync(file);
72+
} catch {
73+
continue; // Not available in this environment.
74+
}
75+
const viaUtf8 = fs.readFileSync(file, 'utf8');
76+
if (file !== '/proc/version' && file.startsWith('/proc/')) {
77+
// Content legitimately differs between two reads; compare shape instead.
78+
assert.ok(viaUtf8.length > 0);
79+
assert.strictEqual(viaUtf8.split('\n').length > 5, true, file);
80+
// Of these, smaps reliably exceeds the 8 KiB stack buffer.
81+
if (file === '/proc/self/smaps') assert.ok(viaUtf8.length > 8192, `smaps is only ${viaUtf8.length} chars`);
82+
} else {
83+
assert.strictEqual(viaUtf8, viaBuffer.toString('utf8'), file);
84+
}
85+
}
86+
}
87+
88+
// Directory: same outcome either way (EISDIR, except on platforms where
89+
// read() accepts directories, e.g. AIX).
90+
function outcome(read) {
91+
try {
92+
return read();
93+
} catch (err) {
94+
return err.code;
95+
}
96+
}
97+
assert.strictEqual(outcome(() => fs.readFileSync(tmpdir.path, 'utf8')),
98+
outcome(() => fs.readFileSync(tmpdir.path).toString('utf8')));

0 commit comments

Comments
0 (0)
You are visitor 01402520 — GeezerVison does not store the pages you look up.
Best viewed with Netscape Navigator 4.7 or Internet Explorer 6 at 800×600. No cookies required.