|
| 1 | +'use strict'; |
| 2 | +// The UTF-8 StringDecoder shares its byte->string conversion with |
| 3 | +// Buffer#toString(): ASCII, Latin-1-representable and general inputs take |
| 4 | +// different (SIMD) paths depending on content and size, and invalid input |
| 5 | +// falls back to a replacing decoder. This test pins the decoder's output for |
| 6 | +// inputs that cross those size thresholds, for chunkings that split multibyte |
| 7 | +// sequences, and for invalid bytes embedded in otherwise large valid input. |
| 8 | +require('../common'); |
| 9 | +const assert = require('assert'); |
| 10 | +const { StringDecoder } = require('string_decoder'); |
| 11 | + |
| 12 | +function decodeInChunks(buf, chunkSize) { |
| 13 | + const decoder = new StringDecoder('utf8'); |
| 14 | + let out = ''; |
| 15 | + for (let i = 0; i < buf.length; i += chunkSize) { |
| 16 | + out += decoder.write(buf.subarray(i, i + chunkSize)); |
| 17 | + } |
| 18 | + return out + decoder.end(); |
| 19 | +} |
| 20 | + |
| 21 | +function check(str, label) { |
| 22 | + const buf = Buffer.from(str, 'utf8'); |
| 23 | + // Sanity: the expectation itself round-trips. |
| 24 | + assert.strictEqual(buf.toString('utf8'), str, `${label}: toString`); |
| 25 | + for (const chunkSize of [1, 2, 3, 4, 5, 7, 31, 32, 33, 255, 256, 257, |
| 26 | + 4095, 4096, 65536, buf.length]) { |
| 27 | + if (chunkSize > buf.length) continue; |
| 28 | + // Keep the test fast: byte-sized chunks only for the smaller inputs. |
| 29 | + if (buf.length > 100_000 && chunkSize < 4095) continue; |
| 30 | + assert.strictEqual(decodeInChunks(buf, chunkSize), str, |
| 31 | + `${label}: chunkSize=${chunkSize}`); |
| 32 | + } |
| 33 | +} |
| 34 | + |
| 35 | +const sizes = [31, 32, 33, 255, 256, 257, 4096, 70000, (1 << 20) + 5]; |
| 36 | +for (const size of sizes) { |
| 37 | + check('a'.repeat(size), `ascii ${size}`); |
| 38 | + // Latin-1 range only (one-byte string in V8, two bytes each in UTF-8). |
| 39 | + check('é'.repeat(size), `latin1 ${size}`); |
| 40 | + // ASCII with a single Latin-1 character at the end / start. |
| 41 | + check('a'.repeat(size - 1) + 'ÿ', `ascii+latin1 tail ${size}`); |
| 42 | + check('Ä' + 'a'.repeat(size - 1), `latin1 head+ascii ${size}`); |
| 43 | + // BMP beyond Latin-1 (three-byte sequences). |
| 44 | + check('日'.repeat(size), `cjk ${size}`); |
| 45 | + // Mixed, including astral plane characters (surrogate pairs, 4 bytes). |
| 46 | + check(('abé日\u{1F600}').repeat(Math.ceil(size / 6)), `mixed ${size}`); |
| 47 | +} |
| 48 | + |
| 49 | +// Invalid bytes inside otherwise valid input of every size class must still be |
| 50 | +// replaced with U+FFFD exactly as before, regardless of chunking. |
| 51 | +for (const size of [8, 40, 300, 5000, (1 << 20) + 5]) { |
| 52 | + const valid = Buffer.from('a'.repeat(size)); |
| 53 | + for (const bad of [[0xff], [0xc0, 0xaf], [0xe2, 0x28, 0xa1], |
| 54 | + [0xed, 0xa0, 0x80] /* encoded surrogate */, |
| 55 | + [0xf0, 0x9f, 0x98] /* truncated 4-byte */]) { |
| 56 | + const buf = Buffer.concat([valid, Buffer.from(bad), valid]); |
| 57 | + const expected = buf.toString('utf8'); |
| 58 | + assert.ok(expected.includes('�'), `size=${size} bad=${bad}`); |
| 59 | + for (const chunkSize of [1, 3, 64, size, size + 1, buf.length]) { |
| 60 | + if (buf.length > 100_000 && chunkSize < size) continue; |
| 61 | + assert.strictEqual(decodeInChunks(buf, chunkSize), expected, |
| 62 | + `invalid ${bad} in ${size}, chunkSize=${chunkSize}`); |
| 63 | + } |
| 64 | + } |
| 65 | +} |
| 66 | + |
| 67 | +// A lone continuation / lead byte split across the size classes at the very |
| 68 | +// end is buffered by the decoder and flushed as U+FFFD by end(). |
| 69 | +{ |
| 70 | + const decoder = new StringDecoder('utf8'); |
| 71 | + const big = Buffer.concat([Buffer.from('a'.repeat(300)), Buffer.from([0xe2, 0x82])]); |
| 72 | + assert.strictEqual(decoder.write(big), 'a'.repeat(300)); |
| 73 | + assert.strictEqual(decoder.end(), '�'); |
| 74 | +} |
| 75 | +{ |
| 76 | + const decoder = new StringDecoder('utf8'); |
| 77 | + const big = Buffer.concat([Buffer.from('é'.repeat(300)), Buffer.from([0xe2, 0x82])]); |
| 78 | + assert.strictEqual(decoder.write(big), 'é'.repeat(300)); |
| 79 | + assert.strictEqual(decoder.write(Buffer.from([0xac])), '€'); |
| 80 | + assert.strictEqual(decoder.end(), ''); |
| 81 | +} |