GeezerVison

The World Wide Web, restored to 800 pixels of glory.

GeezerVison is showing github.com — zlib: add dictionary support to zstdCompress and zstdDecompress · nodejs/node@2013045

File tree

Expand file treeCollapse file tree

Open diff view settings

Filter options

doc/api
- zlib.md
lib
- zlib.js
src
- node_zlib.cc
test/parallel
- test-zlib-zstd-dictionary.js

Expand file treeCollapse file tree

Open diff view settings

Collapse file

`‎doc/api/zlib.md‎`

Copy file name to clipboardExpand all lines: doc/api/zlib.md

+3Lines changed: 3 additions & 0 deletions

Display the source diff
Display the rich diff

Original file line number	Diff line number	Diff line change
@@ -1069,6 +1069,9 @@ Each Zstd-based class takes an `options` object. All options are optional.
`1069`	`1069`	* `maxOutputLength` {integer} Limits output size when using
`1070`	`1070`	[convenience methods][]. Default: [`buffer.kMaxLength`][]
`1071`	`1071`	* `info` {boolean} If `true`, returns an object with `buffer` and `engine`. Default: `false`
	`1072`	+* `dictionary` {Buffer} Optional dictionary used to
	`1073`	`+ improve compression efficiency when compressing or decompressing data that`
	`1074`	`+ shares common patterns with the dictionary.`
`1072`	`1075`
`1073`	`1076`	`For example:`
`1074`	`1077`

Collapse file

`‎lib/zlib.js‎`

Copy file name to clipboardExpand all lines: lib/zlib.js

+3Lines changed: 3 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -893,12 +893,15 @@ class Zstd extends ZlibBase {`
`893`	`893`	`const pledgedSrcSize = opts?.pledgedSrcSize ?? undefined;`
`894`	`894`
`895`	`895`	`const writeState = new Uint32Array(2);`
	`896`	`+`
`896`	`897`	`handle.init(`
`897`	`898`	`initParamsArray,`
`898`	`899`	`pledgedSrcSize,`
`899`	`900`	`writeState,`
`900`	`901`	`processCallback,`
	`902`	`+ opts?.dictionary && isArrayBufferView(opts.dictionary) ? opts.dictionary : undefined,`
`901`	`903`	`);`
	`904`	`+`
`902`	`905`	`super(opts, mode, handle, zstdDefaultOpts);`
`903`	`906`	`this._writeState = writeState;`
`904`	`907`	`}`

Collapse file

`‎src/node_zlib.cc‎`

Copy file name to clipboardExpand all lines: src/node_zlib.cc

+47-7Lines changed: 47 additions & 7 deletions

Original file line number	Diff line number	Diff line change
`@@ -324,7 +324,8 @@ class ZstdCompressContext final : public ZstdContext {`
`324`	`324`	`CompressionError ResetStream();`
`325`	`325`
`326`	`326`	`// Zstd specific:`
`327`		`- CompressionError Init(uint64_t pledged_src_size);`
	`327`	`+ CompressionError Init(uint64_t pledged_src_size,`
	`328`	`+ std::string_view dictionary = {});`
`328`	`329`	`CompressionError SetParameter(int key, int value);`
`329`	`330`
`330`	`331`	`// Wrap ZSTD_freeCCtx to remove the return type.`
`@@ -349,7 +350,9 @@ class ZstdDecompressContext final : public ZstdContext {`
`349`	`350`	`CompressionError ResetStream();`
`350`	`351`
`351`	`352`	`// Zstd specific:`
`352`		`- CompressionError Init(uint64_t pledged_src_size);`
	`353`	`+ CompressionError Init(uint64_t pledged_src_size,`
	`354`	`+ std::string_view dictionary = {});`
	`355`	`+`
`353`	`356`	`CompressionError SetParameter(int key, int value);`
`354`	`357`
`355`	`358`	`// Wrap ZSTD_freeDCtx to remove the return type.`
`@@ -875,8 +878,10 @@ class ZstdStream final : public CompressionStream<CompressionContext> {`
`875`	`878`	`Environment* env = Environment::GetCurrent(args);`
`876`	`879`	`Local<Context> context = env->context();`
`877`	`880`
`878`		`- CHECK(args.Length() == 4 &&`
`879`		`- "init(params, pledgedSrcSize, writeResult, writeCallback)");`
	`881`	`+ CHECK((args.Length() == 4 \|\| args.Length() == 5) &&`
	`882`	`+ "init(params, pledgedSrcSize, writeResult, writeCallback[, "`
	`883`	`+ "dictionary])");`
	`884`	`+`
`880`	`885`	`ZstdStream* wrap;`
`881`	`886`	`ASSIGN_OR_RETURN_UNWRAP(&wrap, args.This());`
`882`	`887`
`@@ -904,7 +909,19 @@ class ZstdStream final : public CompressionStream<CompressionContext> {`
`904`	`909`	`}`
`905`	`910`
`906`	`911`	`AllocScope alloc_scope(wrap);`
`907`		`- CompressionError err = wrap->context()->Init(pledged_src_size);`
	`912`	`+ std::string_view dictionary;`
	`913`	`+ ArrayBufferViewContents<char> contents;`
	`914`	`+ if (args.Length() == 5 && !args[4]->IsUndefined()) {`
	`915`	`+ if (!args[4]->IsArrayBufferView()) {`
	`916`	`+ THROW_ERR_INVALID_ARG_TYPE(`
	`917`	`+ wrap->env(), "dictionary must be an ArrayBufferView if provided");`
	`918`	`+ return;`
	`919`	`+ }`
	`920`	`+ contents.ReadValue(args[4]);`
	`921`	`+ dictionary = std::string_view(contents.data(), contents.length());`
	`922`	`+ }`
	`923`	`+`
	`924`	`+ CompressionError err = wrap->context()->Init(pledged_src_size, dictionary);`
`908`	`925`	`if (err.IsError()) {`
`909`	`926`	`wrap->EmitError(err);`
`910`	`927`	`THROW_ERR_ZLIB_INITIALIZATION_FAILED(wrap->env(), err.message);`
`@@ -1509,14 +1526,26 @@ CompressionError ZstdCompressContext::SetParameter(int key, int value) {`
`1509`	`1526`	`return {};`
`1510`	`1527`	`}`
`1511`	`1528`
`1512`		`-CompressionError ZstdCompressContext::Init(uint64_t pledged_src_size) {`
	`1529`	`+CompressionError ZstdCompressContext::Init(uint64_t pledged_src_size,`
	`1530`	`+ std::string_view dictionary) {`
`1513`	`1531`	`pledged_src_size_ = pledged_src_size;`
`1514`	`1532`	`cctx_.reset(ZSTD_createCCtx());`
`1515`	`1533`	`if (!cctx_) {`
`1516`	`1534`	`return CompressionError("Could not initialize zstd instance",`
`1517`	`1535`	`"ERR_ZLIB_INITIALIZATION_FAILED",`
`1518`	`1536`	`-1);`
`1519`	`1537`	`}`
	`1538`	`+`
	`1539`	`+ if (!dictionary.empty()) {`
	`1540`	`+ size_t ret = ZSTD_CCtx_loadDictionary(`
	`1541`	`+ cctx_.get(), dictionary.data(), dictionary.size());`
	`1542`	`+ if (ZSTD_isError(ret)) {`
	`1543`	`+ return CompressionError("Failed to load zstd dictionary",`
	`1544`	`+ "ERR_ZLIB_DICTIONARY_LOAD_FAILED",`
	`1545`	`+ -1);`
	`1546`	`+ }`
	`1547`	`+ }`
	`1548`	`+`
`1520`	`1549`	`size_t result = ZSTD_CCtx_setPledgedSrcSize(cctx_.get(), pledged_src_size);`
`1521`	`1550`	`if (ZSTD_isError(result)) {`
`1522`	`1551`	`return CompressionError(`
`@@ -1549,13 +1578,24 @@ CompressionError ZstdDecompressContext::SetParameter(int key, int value) {`
`1549`	`1578`	`return {};`
`1550`	`1579`	`}`
`1551`	`1580`
`1552`		`-CompressionError ZstdDecompressContext::Init(uint64_t pledged_src_size) {`
	`1581`	`+CompressionError ZstdDecompressContext::Init(uint64_t pledged_src_size,`
	`1582`	`+ std::string_view dictionary) {`
`1553`	`1583`	`dctx_.reset(ZSTD_createDCtx());`
`1554`	`1584`	`if (!dctx_) {`
`1555`	`1585`	`return CompressionError("Could not initialize zstd instance",`
`1556`	`1586`	`"ERR_ZLIB_INITIALIZATION_FAILED",`
`1557`	`1587`	`-1);`
`1558`	`1588`	`}`
	`1589`	`+`
	`1590`	`+ if (!dictionary.empty()) {`
	`1591`	`+ size_t ret = ZSTD_DCtx_loadDictionary(`
	`1592`	`+ dctx_.get(), dictionary.data(), dictionary.size());`
	`1593`	`+ if (ZSTD_isError(ret)) {`
	`1594`	`+ return CompressionError("Failed to load zstd dictionary",`
	`1595`	`+ "ERR_ZLIB_DICTIONARY_LOAD_FAILED",`
	`1596`	`+ -1);`
	`1597`	`+ }`
	`1598`	`+ }`
`1559`	`1599`	`return {};`
`1560`	`1600`	`}`
`1561`	`1601`

Collapse file

`‎test/parallel/test-zlib-zstd-dictionary.js‎`

Copy file name to clipboard

+26Lines changed: 26 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,26 @@`
	`1`	`+'use strict';`
	`2`	`+`
	`3`	`+const common = require('../common');`
	`4`	`+const assert = require('assert');`
	`5`	`+const zlib = require('zlib');`
	`6`	`+`
	`7`	`+const dictionary = Buffer.from(`
	`8`	+ `Lorem ipsum dolor sit amet, consectetur adipiscing elit.
	`9`	`+ Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.`
	`10`	+ Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.`
	`11`	`+);`
	`12`	`+`
	`13`	`+const input = Buffer.from(`
	`14`	+ `Lorem ipsum dolor sit amet, consectetur adipiscing elit.
	`15`	`+ Lorem ipsum dolor sit amet, consectetur adipiscing elit.`
	`16`	`+ Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.`
	`17`	`+ Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.`
	`18`	+ Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.`
	`19`	`+);`
	`20`	`+`
	`21`	`+zlib.zstdCompress(input, { dictionary }, common.mustSucceed((compressed) => {`
	`22`	`+ assert(compressed.length < input.length);`
	`23`	`+ zlib.zstdDecompress(compressed, { dictionary }, common.mustSucceed((decompressed) => {`
	`24`	`+ assert.strictEqual(decompressed.toString(), input.toString());`
	`25`	`+ }));`
	`26`	`+}));`

0 commit comments

Comments

0 (0)

You are visitor 01402503 — GeezerVison does not store the pages you look up.
Best viewed with Netscape Navigator 4.7 or Internet Explorer 6 at 800×600. No cookies required.