GeezerVison is showing github.com — Refactor tag encoder and decoder to match the YAML specification · nodeca/js-yaml@339d8e0
File treeExpand file treeCollapse file tree Open diff view settings Expand file treeCollapse file tree Open diff view settings Collapse file Copy file name to clipboardExpand all lines: src/common/tagname.ts +83-7Lines changed: 83 additions & 7 deletions | Original file line number | Diff line number | Diff line change |
|---|
@@ -6,35 +6,111 @@ const DEFAULT_TAG_HANDLERS: Readonly<Record<string, string>> = Object.assign( | 6 | 6 | } | 7 | 7 | ) | 8 | 8 |
| 9 | | -function tagPercentEncode (source: string) { | 10 | | - return encodeURI(source).replace(/!/g, '%21') | | 9 | +// YAML 1.2.2, [40] ns-tag-char and [39] ns-uri-char, apart from alphanumerics. | | 10 | +const TAG_SUFFIX_CHARS = "-#;/?:@&=+$_.~*'()" | | 11 | +const TAG_URI_CHARS = TAG_SUFFIX_CHARS + '!,[]' | | 12 | +const HEX = '0123456789ABCDEF' | | 13 | + | | 14 | +function percentByte (byte: number) { | | 15 | + return '%' + HEX[byte >> 4] + HEX[byte & 0x0F] | | 16 | +} | | 17 | + | | 18 | +function tagPercentEncode (source: string, isSuffix: boolean) { | | 19 | + const allowed = isSuffix ? TAG_SUFFIX_CHARS : TAG_URI_CHARS | | 20 | + let result = '' | | 21 | + | | 22 | + for (let index = 0; index < source.length; index++) { | | 23 | + let ch = source.charCodeAt(index) | | 24 | + | | 25 | + if ((ch >= 0x30 && ch <= 0x39) || | | 26 | + (ch >= 0x41 && ch <= 0x5A) || | | 27 | + (ch >= 0x61 && ch <= 0x7A) || | | 28 | + allowed.includes(source[index])) { | | 29 | + result += source[index] | | 30 | + continue | | 31 | + } | | 32 | + | | 33 | + if (ch >= 0xD800 && ch <= 0xDBFF) { | | 34 | + ch = 0x10000 + ((ch & 0x3FF) << 10) + (source.charCodeAt(++index) & 0x3FF) | | 35 | + } | | 36 | + | | 37 | + if (ch < 0x80) { | | 38 | + result += percentByte(ch) | | 39 | + } else if (ch < 0x800) { | | 40 | + result += percentByte(0xC0 | (ch >> 6)) + percentByte(0x80 | (ch & 0x3F)) | | 41 | + } else if (ch < 0x10000) { | | 42 | + result += percentByte(0xE0 | (ch >> 12)) + | | 43 | + percentByte(0x80 | ((ch >> 6) & 0x3F)) + percentByte(0x80 | (ch & 0x3F)) | | 44 | + } else { | | 45 | + result += percentByte(0xF0 | (ch >> 18)) + | | 46 | + percentByte(0x80 | ((ch >> 12) & 0x3F)) + | | 47 | + percentByte(0x80 | ((ch >> 6) & 0x3F)) + percentByte(0x80 | (ch & 0x3F)) | | 48 | + } | | 49 | + } | | 50 | + | | 51 | + return result | | 52 | +} | | 53 | + | | 54 | +function tagPercentDecode (source: string) { | | 55 | + let result = '' | | 56 | + let index = 0 | | 57 | + | | 58 | + // Convert escaped UTF-8 bytes without validating malformed encodings. | | 59 | + while (index < source.length) { | | 60 | + if (source.charCodeAt(index) !== 0x25/* % */) { | | 61 | + result += source[index++] | | 62 | + continue | | 63 | + } | | 64 | + | | 65 | + let ch = parseInt(source.slice(index + 1, index + 3), 16) | | 66 | + index += 3 | | 67 | + | | 68 | + if (ch >= 0xC0) { | | 69 | + let remaining = ch < 0xE0 ? 1 : ch < 0xF0 ? 2 : 3 | | 70 | + ch &= (1 << (6 - remaining)) - 1 | | 71 | + | | 72 | + while (remaining-- > 0) { | | 73 | + ch = (ch << 6) | (parseInt(source.slice(index + 1, index + 3), 16) & 0x3F) | | 74 | + index += 3 | | 75 | + } | | 76 | + } | | 77 | + | | 78 | + if (ch < 0x10000) { | | 79 | + result += String.fromCharCode(ch) | | 80 | + } else { | | 81 | + ch -= 0x10000 | | 82 | + result += String.fromCharCode(0xD800 | (ch >> 10), 0xDC00 | (ch & 0x3FF)) | | 83 | + } | | 84 | + } | | 85 | + | | 86 | + return result | 11 | 87 | } | 12 | 88 |
| 13 | 89 | function tagNameFull (rawTag: string, tagHandlers?: Readonly<Record<string, string>>) { | 14 | 90 | if (rawTag.startsWith('!<') && rawTag.endsWith('>')) { | 15 | | - return decodeURIComponent(rawTag.slice(2, -1)) | | 91 | + return tagPercentDecode(rawTag.slice(2, -1)) | 16 | 92 | } | 17 | 93 |
| 18 | 94 | const handleEnd = rawTag.indexOf('!', 1) | 19 | 95 | const handle = handleEnd === -1 ? '!' : rawTag.slice(0, handleEnd + 1) | 20 | 96 | const prefix = tagHandlers?.[handle] ?? DEFAULT_TAG_HANDLERS[handle] ?? handle | 21 | 97 |
| 22 | | - return decodeURIComponent(prefix) + decodeURIComponent(rawTag.slice(handle.length)) | | 98 | + return tagPercentDecode(prefix) + tagPercentDecode(rawTag.slice(handle.length)) | 23 | 99 | } | 24 | 100 |
| 25 | 101 | function tagNameShort (fullTag: string) { | 26 | 102 | let tag = fullTag | 27 | 103 |
| 28 | 104 | if (tag.charCodeAt(0) === 0x21) { | 29 | 105 | tag = tag.slice(1) | 30 | | - return `!${tagPercentEncode(tag)}` | | 106 | + return `!${tagPercentEncode(tag, true)}` | 31 | 107 | } | 32 | 108 |
| 33 | 109 | if (tag.slice(0, 18) === 'tag:yaml.org,2002:') { | 34 | | - return `!!${tagPercentEncode(tag.slice(18))}` | | 110 | + return `!!${tagPercentEncode(tag.slice(18), true)}` | 35 | 111 | } | 36 | 112 |
| 37 | | - return `!<${tagPercentEncode(tag)}>` | | 113 | + return `!<${tagPercentEncode(tag, false)}>` | 38 | 114 | } | 39 | 115 |
| 40 | 116 | export { | |
Collapse file Copy file name to clipboardExpand all lines: src/parser/parser.ts -10Lines changed: 0 additions & 10 deletions | Original file line number | Diff line number | Diff line change |
|---|
@@ -472,11 +472,6 @@ function readTagProperty (state: ParserState, props: NodeProperties, inFlow: boo | 472 | 472 | if (tagName && !(isVerbatim ? PATTERN_TAG_URI.test(tagName) : PATTERN_TAG_SUFFIX.test(tagName))) { | 473 | 473 | throwError(state, `tag name cannot contain such characters: ${tagName}`) | 474 | 474 | } | 475 | | - try { | 476 | | - decodeURIComponent(tagName) | 477 | | - } catch { | 478 | | - throwError(state, `tag name is malformed: ${tagName}`) | 479 | | - } | 480 | 475 |
| 481 | 476 | if (!isVerbatim && tagHandle !== '!' && tagHandle !== '!!' && !HAS_OWN.call(state.tagHandlers, tagHandle)) { | 482 | 477 | throwError(state, `undeclared tag handle "${tagHandle}"`) | @@ -1367,11 +1362,6 @@ function readDirective (state: ParserState) { | 1367 | 1362 | if (!PATTERN_TAG_HANDLE.test(handle)) throwError(state, 'ill-formed tag handle (first argument) of the TAG directive') | 1368 | 1363 | if (HAS_OWN.call(state.tagHandlers, handle)) throwError(state, `there is a previously declared suffix for "${handle}" tag handle`) | 1369 | 1364 | if (!PATTERN_TAG_PREFIX.test(prefix)) throwError(state, 'ill-formed tag prefix (second argument) of the TAG directive') | 1370 | | - try { | 1371 | | - decodeURIComponent(prefix) | 1372 | | - } catch { | 1373 | | - throwError(state, `tag prefix is malformed: ${prefix}`) | 1374 | | - } | 1375 | 1365 |
| 1376 | 1366 | state.tagHandlers[handle] = prefix | 1377 | 1367 | state.directives.push({ kind: 'tag', handle, prefix }) | |
Collapse file Copy file name to clipboardExpand all lines: test/core/common/tagname.test.mjs +17Lines changed: 17 additions & 0 deletions | Original file line number | Diff line number | Diff line change |
|---|
| 1 | 1 | import { describe, it } from 'node:test' | 2 | 2 | import assert from 'node:assert/strict' | | 3 | +import { parseEvents } from '../../../src/parser/parser.ts' | 3 | 4 | import { | 4 | 5 | tagNameFull, | 5 | 6 | tagNameShort | @@ -27,6 +28,7 @@ describe('tagname', () => { | 27 | 28 | it('resolves a registered custom handle', () => { | 28 | 29 | const handlers = { '!e!': 'tag:example.com,2024:' } | 29 | 30 | assert.equal(tagNameFull('!e!foo', handlers), 'tag:example.com,2024:foo') | | 31 | + assert.equal(tagNameFull('!e!21', { '!e!': '!%25' }), '!%21') | 30 | 32 | }) | 31 | 33 |
| 32 | 34 | it('lets user handles override the defaults', () => { | @@ -42,26 +44,41 @@ describe('tagname', () => { | 42 | 44 | it('percent-decodes the resolved name', () => { | 43 | 45 | assert.equal(tagNameFull('!foo%20bar'), '!foo bar') | 44 | 46 | assert.equal(tagNameFull('!%E2%9C%93'), '!✓') // checkmark | | 47 | + assert.equal(tagNameFull('!%c3%a9%E2%9C%93%f0%9f%98%80+%2521'), '!é✓😀+%21') | 45 | 48 | }) | 46 | 49 | }) | 47 | 50 |
| 48 | 51 | describe('tagNameShort', () => { | 49 | 52 | it('shortens the yaml.org prefix to "!!"', () => { | 50 | 53 | assert.equal(tagNameShort(`${YAML_PREFIX}str`), '!!str') | 51 | 54 | assert.equal(tagNameShort(`${YAML_PREFIX}`), '!!') | | 55 | + assert.equal(tagNameShort(`${YAML_PREFIX}a,b`), '!!a%2Cb') | 52 | 56 | }) | 53 | 57 |
| 54 | 58 | it('keeps a local "!" tag, re-encoding its body', () => { | 55 | 59 | assert.equal(tagNameShort('!local'), '!local') | 56 | 60 | assert.equal(tagNameShort('!foo bar'), '!foo%20bar') | 57 | 61 | assert.equal(tagNameShort('!a!b'), '!a%21b') | | 62 | + assert.equal(tagNameShort('!a.b!,[]{}%'), '!a.b%21%2C%5B%5D%7B%7D%25') | | 63 | + assert.equal(tagNameShort('!é✓😀+%21'), '!%C3%A9%E2%9C%93%F0%9F%98%80+%2521') | 58 | 64 | }) | 59 | 65 |
| 60 | 66 | it('wraps any other tag as verbatim', () => { | 61 | 67 | assert.equal(tagNameShort('tag:example.com,2024:foo'), '!<tag:example.com,2024:foo>') | | 68 | + assert.equal( | | 69 | + tagNameShort('tag:example.com,2024:a!b[c]{}%'), | | 70 | + '!<tag:example.com,2024:a!b[c]%7B%7D%25>' | | 71 | + ) | 62 | 72 | }) | 63 | 73 | }) | 64 | 74 |
| | 75 | + it('does not throw on malformed encodings', () => { | | 76 | + assert.doesNotThrow(() => tagNameShort('!\uD800')) | | 77 | + assert.doesNotThrow(() => tagNameFull('!%FF')) | | 78 | + assert.doesNotThrow(() => parseEvents('!%FF value')) | | 79 | + assert.doesNotThrow(() => parseEvents('%TAG !e! !%FF\n--- !e!foo value')) | | 80 | + }) | | 81 | + | 65 | 82 | describe('round-trips between full and short forms', () => { | 66 | 83 | it('tagNameShort(tagNameFull(x)) recovers the canonical short form', () => { | 67 | 84 | assert.equal(tagNameShort(tagNameFull('!!str')), '!!str') | |
|